telemetry(amazonq): Add changed IDE diagnostics after user acceptance (aws#7130)

leigaol · web-flow · commit fdaaae1a5ba3 · 2025-05-06T15:09:23.000-04:00
## Problem When user accepts suggestion from inline completion or chat, there can be a change in the current open editor's IDE diagnostics, this can be used as a measure for code suggestion quality. Ref: aws/aws-toolkit-jetbrains#5613 This change is part of the server side workspace context. To reduce the risk of large blast radius, this change is only applied for users who are in the experiment of server side project context (both treatment and control), which is the Amzn idc users. E2E verifying request id: `97a3c4d4-0c78-4329-93ae-379d2ec66646`. ## Solution 1. Add changed IDE diagnostics after user acceptance --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license.
diff --git a/packages/amazonq/test/unit/codewhisperer/util/diagnosticsUtil.test.ts b/packages/amazonq/test/unit/codewhisperer/util/diagnosticsUtil.test.ts
@@ -0,0 +1,92 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import * as assert from 'assert'
+import * as vscode from 'vscode'
+import { getDiagnosticsType, getDiagnosticsDifferences } from 'aws-core-vscode/codewhisperer'
+describe('diagnosticsUtil', function () {
+    describe('getDiagnosticsType', function () {
+        it('should identify SYNTAX_ERROR correctly', function () {
+            assert.strictEqual(getDiagnosticsType('Expected semicolon'), 'SYNTAX_ERROR')
+            assert.strictEqual(getDiagnosticsType('Incorrect indent level'), 'SYNTAX_ERROR')
+            assert.strictEqual(getDiagnosticsType('Syntax error in line 5'), 'SYNTAX_ERROR')
+        })
+
+        it('should identify TYPE_ERROR correctly', function () {
+            assert.strictEqual(getDiagnosticsType('Type mismatch'), 'TYPE_ERROR')
+            assert.strictEqual(getDiagnosticsType('Invalid type cast'), 'TYPE_ERROR')
+        })
+
+        it('should identify REFERENCE_ERROR correctly', function () {
+            assert.strictEqual(getDiagnosticsType('Variable is undefined'), 'REFERENCE_ERROR')
+            assert.strictEqual(getDiagnosticsType('Variable not defined'), 'REFERENCE_ERROR')
+            assert.strictEqual(getDiagnosticsType('Reference error occurred'), 'REFERENCE_ERROR')
+        })
+
+        it('should identify BEST_PRACTICE correctly', function () {
+            assert.strictEqual(getDiagnosticsType('Using deprecated method'), 'BEST_PRACTICE')
+            assert.strictEqual(getDiagnosticsType('Variable is unused'), 'BEST_PRACTICE')
+            assert.strictEqual(getDiagnosticsType('Variable not initialized'), 'BEST_PRACTICE')
+        })
+
+        it('should identify SECURITY correctly', function () {
+            assert.strictEqual(getDiagnosticsType('Potential security vulnerability'), 'SECURITY')
+            assert.strictEqual(getDiagnosticsType('Security risk detected'), 'SECURITY')
+        })
+
+        it('should return OTHER for unrecognized messages', function () {
+            assert.strictEqual(getDiagnosticsType('Random message'), 'OTHER')
+            assert.strictEqual(getDiagnosticsType(''), 'OTHER')
+        })
+    })
+
+    describe('getDiagnosticsDifferences', function () {
+        const createDiagnostic = (message: string): vscode.Diagnostic => {
+            return {
+                message,
+                severity: vscode.DiagnosticSeverity.Error,
+                range: new vscode.Range(0, 0, 0, 1),
+                source: 'test',
+            }
+        }
+
+        it('should return empty arrays when both inputs are undefined', function () {
+            const result = getDiagnosticsDifferences(undefined, undefined)
+            assert.deepStrictEqual(result, { added: [], removed: [] })
+        })
+
+        it('should return empty arrays when filepaths are different', function () {
+            const oldDiagnostics = {
+                filepath: '/path/to/file1',
+                diagnostics: [createDiagnostic('error1')],
+            }
+            const newDiagnostics = {
+                filepath: '/path/to/file2',
+                diagnostics: [createDiagnostic('error1')],
+            }
+            const result = getDiagnosticsDifferences(oldDiagnostics, newDiagnostics)
+            assert.deepStrictEqual(result, { added: [], removed: [] })
+        })
+
+        it('should correctly identify added and removed diagnostics', function () {
+            const diagnostic1 = createDiagnostic('error1')
+            const diagnostic2 = createDiagnostic('error2')
+            const diagnostic3 = createDiagnostic('error3')
+
+            const oldDiagnostics = {
+                filepath: '/path/to/file',
+                diagnostics: [diagnostic1, diagnostic2],
+            }
+            const newDiagnostics = {
+                filepath: '/path/to/file',
+                diagnostics: [diagnostic2, diagnostic3],
+            }
+
+            const result = getDiagnosticsDifferences(oldDiagnostics, newDiagnostics)
+            assert.deepStrictEqual(result.added, [diagnostic3])
+            assert.deepStrictEqual(result.removed, [diagnostic1])
+        })
+    })
+})
diff --git a/packages/core/src/amazonq/commons/controllers/contentController.ts b/packages/core/src/amazonq/commons/controllers/contentController.ts
@@ -19,6 +19,7 @@ import { ToolkitError, getErrorMsg } from '../../../shared/errors'
 import fs from '../../../shared/fs/fs'
 import { extractFileAndCodeSelectionFromMessage } from '../../../shared/utilities/textUtilities'
 import { UserWrittenCodeTracker } from '../../../codewhisperer/tracker/userWrittenCodeTracker'
+import { CWCTelemetryHelper } from '../../../codewhispererChat/controllers/chat/telemetryHelper'
 import type { ViewDiff } from '../../../codewhispererChat/controllers/chat/model'
 import type { TriggerEvent } from '../../../codewhispererChat/storages/triggerEvents'
 import { DiffContentProvider } from './diffContentProvider'
@@ -49,6 +50,7 @@ export class EditorContentController {
     ) {
         const editor = window.activeTextEditor
         if (editor) {
+            CWCTelemetryHelper.instance.setDocumentDiagnostics()
             UserWrittenCodeTracker.instance.onQStartsMakingEdits()
             const cursorStart = editor.selection.active
             const indentRange = new vscode.Range(new vscode.Position(cursorStart.line, 0), cursorStart)
diff --git a/packages/core/src/codewhisperer/index.ts b/packages/core/src/codewhisperer/index.ts
@@ -89,6 +89,7 @@ export * from './util/securityScanLanguageContext'
 export * from './util/importAdderUtil'
 export * from './util/globalStateUtil'
 export * from './util/zipUtil'
+export * from './util/diagnosticsUtil'
 export * from './util/commonUtil'
 export * from './util/supplementalContext/codeParsingUtil'
 export * from './util/supplementalContext/supplementalContextUtil'
diff --git a/packages/core/src/codewhisperer/util/codeWhispererSession.ts b/packages/core/src/codewhisperer/util/codeWhispererSession.ts
@@ -2,7 +2,6 @@
  * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
-
 import {
     CodewhispererCompletionType,
     CodewhispererLanguage,
@@ -13,6 +12,7 @@ import {
 import { GenerateRecommendationsRequest, ListRecommendationsRequest, Recommendation } from '../client/codewhisperer'
 import { Position } from 'vscode'
 import { CodeWhispererSupplementalContext, vsCodeState } from '../models/model'
+import { FileDiagnostic, getDiagnosticsOfCurrentFile } from './diagnosticsUtil'
 
 class CodeWhispererSession {
     static #instance: CodeWhispererSession
@@ -45,6 +45,7 @@ class CodeWhispererSession {
     timeToFirstRecommendation = 0
     firstSuggestionShowTime = 0
     perceivedLatency = 0
+    diagnosticsBeforeAccept: FileDiagnostic | undefined = undefined
 
     public static get instance() {
         return (this.#instance ??= new CodeWhispererSession())
@@ -66,6 +67,7 @@ class CodeWhispererSession {
         if (this.invokeSuggestionStartTime) {
             this.timeToFirstRecommendation = timeToFirstRecommendation - this.invokeSuggestionStartTime
         }
+        this.diagnosticsBeforeAccept = getDiagnosticsOfCurrentFile()
     }
 
     setSuggestionState(index: number, value: string) {
@@ -116,6 +118,7 @@ class CodeWhispererSession {
         this.recommendations = []
         this.suggestionStates.clear()
         this.completionTypes.clear()
+        this.diagnosticsBeforeAccept = undefined
     }
 }
 
diff --git a/packages/core/src/codewhisperer/util/diagnosticsUtil.ts b/packages/core/src/codewhisperer/util/diagnosticsUtil.ts
@@ -0,0 +1,117 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import * as vscode from 'vscode'
+import * as crypto from 'crypto'
+import { IdeDiagnostic } from '../client/codewhispereruserclient'
+
+export function getDiagnosticsOfCurrentFile(): FileDiagnostic | undefined {
+    if (vscode.window.activeTextEditor) {
+        return {
+            diagnostics: vscode.languages.getDiagnostics(vscode.window.activeTextEditor.document.uri),
+            filepath: vscode.window.activeTextEditor.document.uri.fsPath,
+        }
+    }
+    return undefined
+}
+
+export type FileDiagnostic = {
+    filepath: string
+    diagnostics: vscode.Diagnostic[]
+}
+
+export function getDiagnosticsDifferences(
+    oldDiagnostics: FileDiagnostic | undefined,
+    newDiagnostics: FileDiagnostic | undefined
+): { added: vscode.Diagnostic[]; removed: vscode.Diagnostic[] } {
+    const result: { added: vscode.Diagnostic[]; removed: vscode.Diagnostic[] } = { added: [], removed: [] }
+    if (
+        oldDiagnostics === undefined ||
+        newDiagnostics === undefined ||
+        newDiagnostics.filepath !== oldDiagnostics.filepath
+    ) {
+        return result
+    }
+
+    // Create maps using diagnostic key for uniqueness
+    const oldMap = new Map(oldDiagnostics.diagnostics.map((d) => [getDiagnosticKey(d), d]))
+    const newMap = new Map(newDiagnostics.diagnostics.map((d) => [getDiagnosticKey(d), d]))
+
+    // Get added diagnostics (in new but not in old)
+    result.added = [...newMap.values()].filter((d) => !oldMap.has(getDiagnosticKey(d)))
+
+    // Get removed diagnostics (in old but not in new)
+    result.removed = [...oldMap.values()].filter((d) => !newMap.has(getDiagnosticKey(d)))
+
+    return result
+}
+
+export function toIdeDiagnostics(diagnostic: vscode.Diagnostic): IdeDiagnostic {
+    const severity =
+        diagnostic.severity === vscode.DiagnosticSeverity.Error
+            ? 'ERROR'
+            : diagnostic.severity === vscode.DiagnosticSeverity.Warning
+              ? 'WARNING'
+              : diagnostic.severity === vscode.DiagnosticSeverity.Hint
+                ? 'HINT'
+                : 'INFORMATION'
+
+    return {
+        ideDiagnosticType: getDiagnosticsType(diagnostic.message),
+        severity: severity,
+        source: diagnostic.source,
+        range: {
+            start: {
+                line: diagnostic.range.start.line,
+                character: diagnostic.range.start.character,
+            },
+            end: {
+                line: diagnostic.range.end.line,
+                character: diagnostic.range.end.character,
+            },
+        },
+    }
+}
+
+export function getDiagnosticsType(message: string): string {
+    const errorTypes = new Map([
+        ['SYNTAX_ERROR', ['expected', 'indent', 'syntax']],
+        ['TYPE_ERROR', ['type', 'cast']],
+        ['REFERENCE_ERROR', ['undefined', 'not defined', 'undeclared', 'reference']],
+        ['BEST_PRACTICE', ['deprecated', 'unused', 'uninitialized', 'not initialized']],
+        ['SECURITY', ['security', 'vulnerability']],
+    ])
+
+    const lowercaseMessage = message.toLowerCase()
+
+    for (const [errorType, keywords] of errorTypes) {
+        if (keywords.some((keyword) => lowercaseMessage.includes(keyword))) {
+            return errorType
+        }
+    }
+
+    return 'OTHER'
+}
+
+/**
+ * Generates a unique MD5 hash key for a VS Code diagnostic object.
+ *
+ * @param diagnostic - A VS Code Diagnostic object containing information about a code diagnostic
+ * @returns A 32-character hexadecimal MD5 hash string that uniquely identifies the diagnostic
+ *
+ * @description
+ * Creates a deterministic hash by combining the diagnostic's message, severity, code, and source.
+ * This hash can be used as a unique identifier for deduplication or tracking purposes.
+ * Note: range is not in the hashed string because a diagnostic can move and its range can change within the editor
+ */
+function getDiagnosticKey(diagnostic: vscode.Diagnostic): string {
+    const jsonStr = JSON.stringify({
+        message: diagnostic.message,
+        severity: diagnostic.severity,
+        code: diagnostic.code,
+        source: diagnostic.source,
+    })
+
+    return crypto.createHash('md5').update(jsonStr).digest('hex')
+}
diff --git a/packages/core/src/codewhisperer/util/telemetryHelper.ts b/packages/core/src/codewhisperer/util/telemetryHelper.ts
@@ -26,9 +26,12 @@ import { getLogger } from '../../shared/logger/logger'
 import { session } from './codeWhispererSession'
 import { CodeWhispererSupplementalContext } from '../models/model'
 import { FeatureConfigProvider } from '../../shared/featureConfig'
-import { CodeScanRemediationsEventType } from '../client/codewhispereruserclient'
+import CodeWhispererUserClient, { CodeScanRemediationsEventType } from '../client/codewhispereruserclient'
 import { CodeAnalysisScope as CodeAnalysisScopeClientSide } from '../models/constants'
 import { Session } from '../../amazonqTest/chat/session/session'
+import { sleep } from '../../shared/utilities/timeoutUtils'
+import { getDiagnosticsDifferences, getDiagnosticsOfCurrentFile, toIdeDiagnostics } from './diagnosticsUtil'
+import { Auth } from '../../auth/auth'
 
 export class TelemetryHelper {
     // Some variables for client component latency
@@ -422,46 +425,56 @@ export class TelemetryHelper {
             e2eLatency = 0.0
         }
 
-        client
-            .sendTelemetryEvent({
-                telemetryEvent: {
-                    userTriggerDecisionEvent: {
-                        sessionId: sessionId,
-                        requestId: this.sessionDecisions[0].codewhispererFirstRequestId,
-                        customizationArn: selectedCustomization.arn === '' ? undefined : selectedCustomization.arn,
-                        programmingLanguage: {
-                            languageName: runtimeLanguageContext.toRuntimeLanguage(
-                                this.sessionDecisions[0].codewhispererLanguage
-                            ),
-                        },
-                        completionType: this.getSendTelemetryCompletionType(aggregatedCompletionType),
-                        suggestionState: this.getSendTelemetrySuggestionState(aggregatedSuggestionState),
-                        recommendationLatencyMilliseconds: e2eLatency,
-                        triggerToResponseLatencyMilliseconds: session.timeToFirstRecommendation,
-                        perceivedLatencyMilliseconds: session.perceivedLatency,
-                        timestamp: new Date(Date.now()),
-                        suggestionReferenceCount: referenceCount,
-                        generatedLine: generatedLines,
-                        numberOfRecommendations: suggestionCount,
-                        acceptedCharacterCount: acceptedRecommendationContent.length,
-                    },
-                },
-                profileArn: profile?.arn,
-            })
-            .then()
-            .catch((error) => {
-                let requestId: string | undefined
-                if (isAwsError(error)) {
-                    requestId = error.requestId
-                }
+        const userTriggerDecisionEvent: CodeWhispererUserClient.UserTriggerDecisionEvent = {
+            sessionId: sessionId,
+            requestId: this.sessionDecisions[0].codewhispererFirstRequestId,
+            customizationArn: selectedCustomization.arn === '' ? undefined : selectedCustomization.arn,
+            programmingLanguage: {
+                languageName: runtimeLanguageContext.toRuntimeLanguage(this.sessionDecisions[0].codewhispererLanguage),
+            },
+            completionType: this.getSendTelemetryCompletionType(aggregatedCompletionType),
+            suggestionState: this.getSendTelemetrySuggestionState(aggregatedSuggestionState),
+            recommendationLatencyMilliseconds: e2eLatency,
+            triggerToResponseLatencyMilliseconds: session.timeToFirstRecommendation,
+            perceivedLatencyMilliseconds: session.perceivedLatency,
+            timestamp: new Date(Date.now()),
+            suggestionReferenceCount: referenceCount,
+            generatedLine: generatedLines,
+            numberOfRecommendations: suggestionCount,
+            acceptedCharacterCount: acceptedRecommendationContent.length,
+        }
+        this.resetUserTriggerDecisionTelemetry()
 
-                getLogger().debug(
-                    `Failed to sendTelemetryEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${
-                        error.message
-                    }`
+        const sendEvent = () =>
+            client
+                .sendTelemetryEvent({
+                    telemetryEvent: { userTriggerDecisionEvent: userTriggerDecisionEvent },
+                    profileArn: profile?.arn,
+                })
+                .catch((error) => {
+                    const requestId = isAwsError(error) ? error.requestId : undefined
+                    getLogger().debug(
+                        `Failed to sendTelemetryEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${error.message}`
+                    )
+                })
+
+        if (userTriggerDecisionEvent.suggestionState === 'ACCEPT' && Auth.instance.isInternalAmazonUser()) {
+            // wait 1 seconds for the user installed 3rd party LSP
+            // to update its diagnostics.
+            void sleep(1000).then(() => {
+                const diagnosticDiff = getDiagnosticsDifferences(
+                    session.diagnosticsBeforeAccept,
+                    getDiagnosticsOfCurrentFile()
+                )
+                userTriggerDecisionEvent.addedIdeDiagnostics = diagnosticDiff.added.map((it) => toIdeDiagnostics(it))
+                userTriggerDecisionEvent.removedIdeDiagnostics = diagnosticDiff.removed.map((it) =>
+                    toIdeDiagnostics(it)
                 )
+                void sendEvent()
             })
-        this.resetUserTriggerDecisionTelemetry()
+        } else {
+            void sendEvent()
+        }
     }
 
     public getLastTriggerDecisionForClassifier() {
diff --git a/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts b/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts
diff --git a/packages/core/src/test/amazonq/common/contentController.test.ts b/packages/core/src/test/amazonq/common/contentController.test.ts
