telemetry(amazonq): Add changed IDE diagnostics after user acceptance

packages/amazonq/test/unit/codewhisperer/util/diagnosticsUtil.test.ts

@@ -0,0 +1,92 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import * as assert from 'assert'
+import * as vscode from 'vscode'
+import { getDiagnosticsType, getDiagnosticsDifferences } from 'aws-core-vscode/codewhisperer'
+describe('diagnosticsUtil', function () {
+    describe('getDiagnosticsType', function () {
+        it('should identify SYNTAX_ERROR correctly', function () {
+            assert.strictEqual(getDiagnosticsType('Expected semicolon'), 'SYNTAX_ERROR')
+            assert.strictEqual(getDiagnosticsType('Incorrect indent level'), 'SYNTAX_ERROR')
+            assert.strictEqual(getDiagnosticsType('Syntax error in line 5'), 'SYNTAX_ERROR')
+        })
+
+        it('should identify TYPE_ERROR correctly', function () {
+            assert.strictEqual(getDiagnosticsType('Type mismatch'), 'TYPE_ERROR')
+            assert.strictEqual(getDiagnosticsType('Invalid type cast'), 'TYPE_ERROR')
+        })
+
+        it('should identify REFERENCE_ERROR correctly', function () {
+            assert.strictEqual(getDiagnosticsType('Variable is undefined'), 'REFERENCE_ERROR')
+            assert.strictEqual(getDiagnosticsType('Variable not defined'), 'REFERENCE_ERROR')
+            assert.strictEqual(getDiagnosticsType('Reference error occurred'), 'REFERENCE_ERROR')
+        })
+
+        it('should identify BEST_PRACTICE correctly', function () {
+            assert.strictEqual(getDiagnosticsType('Using deprecated method'), 'BEST_PRACTICE')
+            assert.strictEqual(getDiagnosticsType('Variable is unused'), 'BEST_PRACTICE')
+            assert.strictEqual(getDiagnosticsType('Variable not initialized'), 'BEST_PRACTICE')
+        })
+
+        it('should identify SECURITY correctly', function () {
+            assert.strictEqual(getDiagnosticsType('Potential security vulnerability'), 'SECURITY')
+            assert.strictEqual(getDiagnosticsType('Security risk detected'), 'SECURITY')
+        })
+
+        it('should return OTHER for unrecognized messages', function () {
+            assert.strictEqual(getDiagnosticsType('Random message'), 'OTHER')
+            assert.strictEqual(getDiagnosticsType(''), 'OTHER')
+        })
+    })
+
+    describe('getDiagnosticsDifferences', function () {
+        const createDiagnostic = (message: string): vscode.Diagnostic => {
+            return {
+                message,
+                severity: vscode.DiagnosticSeverity.Error,
+                range: new vscode.Range(0, 0, 0, 1),
+                source: 'test',
+            }
+        }
+
+        it('should return empty arrays when both inputs are undefined', function () {
+            const result = getDiagnosticsDifferences(undefined, undefined)
+            assert.deepStrictEqual(result, { added: [], removed: [] })
+        })
+
+        it('should return empty arrays when filepaths are different', function () {
+            const oldDiagnostics = {
+                filepath: '/path/to/file1',
+                diagnostics: [createDiagnostic('error1')],
+            }
+            const newDiagnostics = {
+                filepath: '/path/to/file2',
+                diagnostics: [createDiagnostic('error1')],
+            }
+            const result = getDiagnosticsDifferences(oldDiagnostics, newDiagnostics)
+            assert.deepStrictEqual(result, { added: [], removed: [] })
+        })
+
+        it('should correctly identify added and removed diagnostics', function () {
+            const diagnostic1 = createDiagnostic('error1')
+            const diagnostic2 = createDiagnostic('error2')
+            const diagnostic3 = createDiagnostic('error3')
+
+            const oldDiagnostics = {
+                filepath: '/path/to/file',
+                diagnostics: [diagnostic1, diagnostic2],
+            }
+            const newDiagnostics = {
+                filepath: '/path/to/file',
+                diagnostics: [diagnostic2, diagnostic3],
+            }
+
+            const result = getDiagnosticsDifferences(oldDiagnostics, newDiagnostics)
+            assert.deepStrictEqual(result.added, [diagnostic3])
+            assert.deepStrictEqual(result.removed, [diagnostic1])
+        })
+    })
+})

packages/core/src/amazonq/commons/controllers/contentController.ts

@@ -19,6 +19,7 @@ import { ToolkitError, getErrorMsg } from '../../../shared/errors'
 import fs from '../../../shared/fs/fs'
 import { extractFileAndCodeSelectionFromMessage } from '../../../shared/utilities/textUtilities'
 import { UserWrittenCodeTracker } from '../../../codewhisperer/tracker/userWrittenCodeTracker'
+import { CWCTelemetryHelper } from '../../../codewhispererChat/controllers/chat/telemetryHelper'
 
 export class ContentProvider implements vscode.TextDocumentContentProvider {
     constructor(private uri: vscode.Uri) {}
@@ -44,6 +45,7 @@ export class EditorContentController {
     ) {
         const editor = window.activeTextEditor
         if (editor) {
+            CWCTelemetryHelper.instance.setDocumentDiagnostics()
             UserWrittenCodeTracker.instance.onQStartsMakingEdits()
             const cursorStart = editor.selection.active
             const indentRange = new vscode.Range(new vscode.Position(cursorStart.line, 0), cursorStart)

packages/core/src/codewhisperer/index.ts

@@ -89,6 +89,7 @@ export * from './util/securityScanLanguageContext'
 export * from './util/importAdderUtil'
 export * from './util/globalStateUtil'
 export * from './util/zipUtil'
+export * from './util/diagnosticsUtil'
 export * from './util/commonUtil'
 export * from './util/supplementalContext/codeParsingUtil'
 export * from './util/supplementalContext/supplementalContextUtil'

packages/core/src/codewhisperer/util/codeWhispererSession.ts

@@ -2,7 +2,6 @@
  * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
-
 import {
     CodewhispererCompletionType,
     CodewhispererLanguage,
@@ -13,6 +12,7 @@ import {
 import { GenerateRecommendationsRequest, ListRecommendationsRequest, Recommendation } from '../client/codewhisperer'
 import { Position } from 'vscode'
 import { CodeWhispererSupplementalContext, vsCodeState } from '../models/model'
+import { FileDiagnostic, getDiagnosticsOfCurrentFile } from './diagnosticsUtil'
 
 class CodeWhispererSession {
     static #instance: CodeWhispererSession
@@ -45,6 +45,7 @@ class CodeWhispererSession {
     timeToFirstRecommendation = 0
     firstSuggestionShowTime = 0
     perceivedLatency = 0
+    diagnosticsBeforeAccept: FileDiagnostic | undefined = undefined
 
     public static get instance() {
         return (this.#instance ??= new CodeWhispererSession())
@@ -66,6 +67,7 @@ class CodeWhispererSession {
         if (this.invokeSuggestionStartTime) {
             this.timeToFirstRecommendation = timeToFirstRecommendation - this.invokeSuggestionStartTime
         }
+        this.diagnosticsBeforeAccept = getDiagnosticsOfCurrentFile()
     }
 
     setSuggestionState(index: number, value: string) {
@@ -116,6 +118,7 @@ class CodeWhispererSession {
         this.recommendations = []
         this.suggestionStates.clear()
         this.completionTypes.clear()
+        this.diagnosticsBeforeAccept = undefined
     }
 }
 

packages/core/src/codewhisperer/util/diagnosticsUtil.ts

@@ -0,0 +1,117 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import * as vscode from 'vscode'
+import * as crypto from 'crypto'
+import { IdeDiagnostic } from '../client/codewhispereruserclient'
+
+export function getDiagnosticsOfCurrentFile(): FileDiagnostic | undefined {
+    if (vscode.window.activeTextEditor) {
+        return {
+            diagnostics: vscode.languages.getDiagnostics(vscode.window.activeTextEditor.document.uri),
+            filepath: vscode.window.activeTextEditor.document.uri.fsPath,
+        }
+    }
+    return undefined
+}
+
+export type FileDiagnostic = {
+    filepath: string
+    diagnostics: vscode.Diagnostic[]
+}
+
+export function getDiagnosticsDifferences(
+    oldDiagnostics: FileDiagnostic | undefined,
+    newDiagnostics: FileDiagnostic | undefined
+): { added: vscode.Diagnostic[]; removed: vscode.Diagnostic[] } {
+    const result: { added: vscode.Diagnostic[]; removed: vscode.Diagnostic[] } = { added: [], removed: [] }
+    if (
+        oldDiagnostics === undefined ||
+        newDiagnostics === undefined ||
+        newDiagnostics.filepath !== oldDiagnostics.filepath
+    ) {
+        return result
+    }
+
+    // Create maps using diagnostic key for uniqueness
+    const oldMap = new Map(oldDiagnostics.diagnostics.map((d) => [getDiagnosticKey(d), d]))
+    const newMap = new Map(newDiagnostics.diagnostics.map((d) => [getDiagnosticKey(d), d]))
+
+    // Get added diagnostics (in new but not in old)
+    result.added = [...newMap.values()].filter((d) => !oldMap.has(getDiagnosticKey(d)))
+
+    // Get removed diagnostics (in old but not in new)
+    result.removed = [...oldMap.values()].filter((d) => !newMap.has(getDiagnosticKey(d)))
+
+    return result
+}
+
+export function toIdeDiagnostics(diagnostic: vscode.Diagnostic): IdeDiagnostic {
+    const severity =
+        diagnostic.severity === vscode.DiagnosticSeverity.Error
+            ? 'ERROR'
+            : diagnostic.severity === vscode.DiagnosticSeverity.Warning
+              ? 'WARNING'
+              : diagnostic.severity === vscode.DiagnosticSeverity.Hint
+                ? 'HINT'
+                : 'INFORMATION'
+
+    return {
+        ideDiagnosticType: getDiagnosticsType(diagnostic.message),
+        severity: severity,
+        source: diagnostic.source,
+        range: {
+            start: {
+                line: diagnostic.range.start.line,
+                character: diagnostic.range.start.character,
+            },
+            end: {
+                line: diagnostic.range.end.line,
+                character: diagnostic.range.end.character,
+            },
+        },
+    }
+}
+
+export function getDiagnosticsType(message: string): string {
+    const errorTypes = new Map([
+        ['SYNTAX_ERROR', ['expected', 'indent', 'syntax']],
+        ['TYPE_ERROR', ['type', 'cast']],
+        ['REFERENCE_ERROR', ['undefined', 'not defined', 'undeclared', 'reference']],
+        ['BEST_PRACTICE', ['deprecated', 'unused', 'uninitialized', 'not initialized']],
+        ['SECURITY', ['security', 'vulnerability']],
+    ])
+
+    const lowercaseMessage = message.toLowerCase()
+
+    for (const [errorType, keywords] of errorTypes) {
+        if (keywords.some((keyword) => lowercaseMessage.includes(keyword))) {
+            return errorType
+        }
+    }
+
+    return 'OTHER'
+}
+
+/**
+ * Generates a unique MD5 hash key for a VS Code diagnostic object.
+ *
+ * @param diagnostic - A VS Code Diagnostic object containing information about a code diagnostic
+ * @returns A 32-character hexadecimal MD5 hash string that uniquely identifies the diagnostic
+ *
+ * @description
+ * Creates a deterministic hash by combining the diagnostic's message, severity, code, and source.
+ * This hash can be used as a unique identifier for deduplication or tracking purposes.
+ * Note: range is not in the hashed string because a diagnostic can move and its range can change within the editor
+ */
+function getDiagnosticKey(diagnostic: vscode.Diagnostic): string {
+    const jsonStr = JSON.stringify({
+        message: diagnostic.message,
+        severity: diagnostic.severity,
+        code: diagnostic.code,
+        source: diagnostic.source,
+    })
+
+    return crypto.createHash('md5').update(jsonStr).digest('hex')
+}

packages/core/src/codewhisperer/util/telemetryHelper.ts

@@ -26,9 +26,12 @@ import { getLogger } from '../../shared/logger/logger'
 import { session } from './codeWhispererSession'
 import { CodeWhispererSupplementalContext } from '../models/model'
 import { FeatureConfigProvider } from '../../shared/featureConfig'
-import { CodeScanRemediationsEventType } from '../client/codewhispereruserclient'
+import CodeWhispererUserClient, { CodeScanRemediationsEventType } from '../client/codewhispereruserclient'
 import { CodeAnalysisScope as CodeAnalysisScopeClientSide } from '../models/constants'
 import { Session } from '../../amazonqTest/chat/session/session'
+import { sleep } from '../../shared/utilities/timeoutUtils'
+import { getDiagnosticsDifferences, getDiagnosticsOfCurrentFile, toIdeDiagnostics } from './diagnosticsUtil'
+import { Auth } from '../../auth/auth'
 
 export class TelemetryHelper {
     // Some variables for client component latency
@@ -422,46 +425,56 @@ export class TelemetryHelper {
             e2eLatency = 0.0
         }
 
-        client
-            .sendTelemetryEvent({
-                telemetryEvent: {
-                    userTriggerDecisionEvent: {
-                        sessionId: sessionId,
-                        requestId: this.sessionDecisions[0].codewhispererFirstRequestId,
-                        customizationArn: selectedCustomization.arn === '' ? undefined : selectedCustomization.arn,
-                        programmingLanguage: {
-                            languageName: runtimeLanguageContext.toRuntimeLanguage(
-                                this.sessionDecisions[0].codewhispererLanguage
-                            ),
-                        },
-                        completionType: this.getSendTelemetryCompletionType(aggregatedCompletionType),
-                        suggestionState: this.getSendTelemetrySuggestionState(aggregatedSuggestionState),
-                        recommendationLatencyMilliseconds: e2eLatency,
-                        triggerToResponseLatencyMilliseconds: session.timeToFirstRecommendation,
-                        perceivedLatencyMilliseconds: session.perceivedLatency,
-                        timestamp: new Date(Date.now()),
-                        suggestionReferenceCount: referenceCount,
-                        generatedLine: generatedLines,
-                        numberOfRecommendations: suggestionCount,
-                        acceptedCharacterCount: acceptedRecommendationContent.length,
-                    },
-                },
-                profileArn: profile?.arn,
-            })
-            .then()
-            .catch((error) => {
-                let requestId: string | undefined
-                if (isAwsError(error)) {
-                    requestId = error.requestId
-                }
+        const userTriggerDecisionEvent: CodeWhispererUserClient.UserTriggerDecisionEvent = {
+            sessionId: sessionId,
+            requestId: this.sessionDecisions[0].codewhispererFirstRequestId,
+            customizationArn: selectedCustomization.arn === '' ? undefined : selectedCustomization.arn,
+            programmingLanguage: {
+                languageName: runtimeLanguageContext.toRuntimeLanguage(this.sessionDecisions[0].codewhispererLanguage),
+            },
+            completionType: this.getSendTelemetryCompletionType(aggregatedCompletionType),
+            suggestionState: this.getSendTelemetrySuggestionState(aggregatedSuggestionState),
+            recommendationLatencyMilliseconds: e2eLatency,
+            triggerToResponseLatencyMilliseconds: session.timeToFirstRecommendation,
+            perceivedLatencyMilliseconds: session.perceivedLatency,
+            timestamp: new Date(Date.now()),
+            suggestionReferenceCount: referenceCount,
+            generatedLine: generatedLines,
+            numberOfRecommendations: suggestionCount,
+            acceptedCharacterCount: acceptedRecommendationContent.length,
+        }
+        this.resetUserTriggerDecisionTelemetry()
 
-                getLogger().debug(
-                    `Failed to sendTelemetryEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${
-                        error.message
-                    }`
+        const sendEvent = () =>
+            client
+                .sendTelemetryEvent({
+                    telemetryEvent: { userTriggerDecisionEvent: userTriggerDecisionEvent },
+                    profileArn: profile?.arn,
+                })
+                .catch((error) => {
+                    const requestId = isAwsError(error) ? error.requestId : undefined
+                    getLogger().debug(
+                        `Failed to sendTelemetryEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${error.message}`
+                    )
+                })
+
+        if (userTriggerDecisionEvent.suggestionState === 'ACCEPT' && Auth.instance.isInternalAmazonUser()) {
+            // wait 1 seconds for the user installed 3rd party LSP
+            // to update its diagnostics.
+            void sleep(1000).then(() => {
+                const diagnosticDiff = getDiagnosticsDifferences(
+                    session.diagnosticsBeforeAccept,
+                    getDiagnosticsOfCurrentFile()
+                )
+                userTriggerDecisionEvent.addedIdeDiagnostics = diagnosticDiff.added.map((it) => toIdeDiagnostics(it))
+                userTriggerDecisionEvent.removedIdeDiagnostics = diagnosticDiff.removed.map((it) =>
+                    toIdeDiagnostics(it)
                 )
+                void sendEvent()
             })
-        this.resetUserTriggerDecisionTelemetry()
+        } else {
+            void sendEvent()
+        }
     }
 
     public getLastTriggerDecisionForClassifier() {