diff --git a/packages/amazonq/test/unit/codewhisperer/util/diagnosticsUtil.test.ts b/packages/amazonq/test/unit/codewhisperer/util/diagnosticsUtil.test.ts new file mode 100644 index 00000000000..6c33d61ba87 --- /dev/null +++ b/packages/amazonq/test/unit/codewhisperer/util/diagnosticsUtil.test.ts @@ -0,0 +1,92 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as assert from 'assert' +import * as vscode from 'vscode' +import { getDiagnosticsType, getDiagnosticsDifferences } from 'aws-core-vscode/codewhisperer' +describe('diagnosticsUtil', function () { + describe('getDiagnosticsType', function () { + it('should identify SYNTAX_ERROR correctly', function () { + assert.strictEqual(getDiagnosticsType('Expected semicolon'), 'SYNTAX_ERROR') + assert.strictEqual(getDiagnosticsType('Incorrect indent level'), 'SYNTAX_ERROR') + assert.strictEqual(getDiagnosticsType('Syntax error in line 5'), 'SYNTAX_ERROR') + }) + + it('should identify TYPE_ERROR correctly', function () { + assert.strictEqual(getDiagnosticsType('Type mismatch'), 'TYPE_ERROR') + assert.strictEqual(getDiagnosticsType('Invalid type cast'), 'TYPE_ERROR') + }) + + it('should identify REFERENCE_ERROR correctly', function () { + assert.strictEqual(getDiagnosticsType('Variable is undefined'), 'REFERENCE_ERROR') + assert.strictEqual(getDiagnosticsType('Variable not defined'), 'REFERENCE_ERROR') + assert.strictEqual(getDiagnosticsType('Reference error occurred'), 'REFERENCE_ERROR') + }) + + it('should identify BEST_PRACTICE correctly', function () { + assert.strictEqual(getDiagnosticsType('Using deprecated method'), 'BEST_PRACTICE') + assert.strictEqual(getDiagnosticsType('Variable is unused'), 'BEST_PRACTICE') + assert.strictEqual(getDiagnosticsType('Variable not initialized'), 'BEST_PRACTICE') + }) + + it('should identify SECURITY correctly', function () { + assert.strictEqual(getDiagnosticsType('Potential security vulnerability'), 'SECURITY') + assert.strictEqual(getDiagnosticsType('Security risk detected'), 'SECURITY') + }) + + it('should return OTHER for unrecognized messages', function () { + assert.strictEqual(getDiagnosticsType('Random message'), 'OTHER') + assert.strictEqual(getDiagnosticsType(''), 'OTHER') + }) + }) + + describe('getDiagnosticsDifferences', function () { + const createDiagnostic = (message: string): vscode.Diagnostic => { + return { + message, + severity: vscode.DiagnosticSeverity.Error, + range: new vscode.Range(0, 0, 0, 1), + source: 'test', + } + } + + it('should return empty arrays when both inputs are undefined', function () { + const result = getDiagnosticsDifferences(undefined, undefined) + assert.deepStrictEqual(result, { added: [], removed: [] }) + }) + + it('should return empty arrays when filepaths are different', function () { + const oldDiagnostics = { + filepath: '/path/to/file1', + diagnostics: [createDiagnostic('error1')], + } + const newDiagnostics = { + filepath: '/path/to/file2', + diagnostics: [createDiagnostic('error1')], + } + const result = getDiagnosticsDifferences(oldDiagnostics, newDiagnostics) + assert.deepStrictEqual(result, { added: [], removed: [] }) + }) + + it('should correctly identify added and removed diagnostics', function () { + const diagnostic1 = createDiagnostic('error1') + const diagnostic2 = createDiagnostic('error2') + const diagnostic3 = createDiagnostic('error3') + + const oldDiagnostics = { + filepath: '/path/to/file', + diagnostics: [diagnostic1, diagnostic2], + } + const newDiagnostics = { + filepath: '/path/to/file', + diagnostics: [diagnostic2, diagnostic3], + } + + const result = getDiagnosticsDifferences(oldDiagnostics, newDiagnostics) + assert.deepStrictEqual(result.added, [diagnostic3]) + assert.deepStrictEqual(result.removed, [diagnostic1]) + }) + }) +}) diff --git a/packages/core/src/amazonq/commons/controllers/contentController.ts b/packages/core/src/amazonq/commons/controllers/contentController.ts index 70744417451..97c828e6f9e 100644 --- a/packages/core/src/amazonq/commons/controllers/contentController.ts +++ b/packages/core/src/amazonq/commons/controllers/contentController.ts @@ -19,6 +19,7 @@ import { ToolkitError, getErrorMsg } from '../../../shared/errors' import fs from '../../../shared/fs/fs' import { extractFileAndCodeSelectionFromMessage } from '../../../shared/utilities/textUtilities' import { UserWrittenCodeTracker } from '../../../codewhisperer/tracker/userWrittenCodeTracker' +import { CWCTelemetryHelper } from '../../../codewhispererChat/controllers/chat/telemetryHelper' import type { ViewDiff } from '../../../codewhispererChat/controllers/chat/model' import type { TriggerEvent } from '../../../codewhispererChat/storages/triggerEvents' import { DiffContentProvider } from './diffContentProvider' @@ -49,6 +50,7 @@ export class EditorContentController { ) { const editor = window.activeTextEditor if (editor) { + CWCTelemetryHelper.instance.setDocumentDiagnostics() UserWrittenCodeTracker.instance.onQStartsMakingEdits() const cursorStart = editor.selection.active const indentRange = new vscode.Range(new vscode.Position(cursorStart.line, 0), cursorStart) diff --git a/packages/core/src/codewhisperer/index.ts b/packages/core/src/codewhisperer/index.ts index 930b168beec..b2e616ac0ac 100644 --- a/packages/core/src/codewhisperer/index.ts +++ b/packages/core/src/codewhisperer/index.ts @@ -89,6 +89,7 @@ export * from './util/securityScanLanguageContext' export * from './util/importAdderUtil' export * from './util/globalStateUtil' export * from './util/zipUtil' +export * from './util/diagnosticsUtil' export * from './util/commonUtil' export * from './util/supplementalContext/codeParsingUtil' export * from './util/supplementalContext/supplementalContextUtil' diff --git a/packages/core/src/codewhisperer/util/codeWhispererSession.ts b/packages/core/src/codewhisperer/util/codeWhispererSession.ts index 042cd947124..17d9c998112 100644 --- a/packages/core/src/codewhisperer/util/codeWhispererSession.ts +++ b/packages/core/src/codewhisperer/util/codeWhispererSession.ts @@ -2,7 +2,6 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ - import { CodewhispererCompletionType, CodewhispererLanguage, @@ -13,6 +12,7 @@ import { import { GenerateRecommendationsRequest, ListRecommendationsRequest, Recommendation } from '../client/codewhisperer' import { Position } from 'vscode' import { CodeWhispererSupplementalContext, vsCodeState } from '../models/model' +import { FileDiagnostic, getDiagnosticsOfCurrentFile } from './diagnosticsUtil' class CodeWhispererSession { static #instance: CodeWhispererSession @@ -45,6 +45,7 @@ class CodeWhispererSession { timeToFirstRecommendation = 0 firstSuggestionShowTime = 0 perceivedLatency = 0 + diagnosticsBeforeAccept: FileDiagnostic | undefined = undefined public static get instance() { return (this.#instance ??= new CodeWhispererSession()) @@ -66,6 +67,7 @@ class CodeWhispererSession { if (this.invokeSuggestionStartTime) { this.timeToFirstRecommendation = timeToFirstRecommendation - this.invokeSuggestionStartTime } + this.diagnosticsBeforeAccept = getDiagnosticsOfCurrentFile() } setSuggestionState(index: number, value: string) { @@ -116,6 +118,7 @@ class CodeWhispererSession { this.recommendations = [] this.suggestionStates.clear() this.completionTypes.clear() + this.diagnosticsBeforeAccept = undefined } } diff --git a/packages/core/src/codewhisperer/util/diagnosticsUtil.ts b/packages/core/src/codewhisperer/util/diagnosticsUtil.ts new file mode 100644 index 00000000000..5a34c3e4430 --- /dev/null +++ b/packages/core/src/codewhisperer/util/diagnosticsUtil.ts @@ -0,0 +1,117 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import * as vscode from 'vscode' +import * as crypto from 'crypto' +import { IdeDiagnostic } from '../client/codewhispereruserclient' + +export function getDiagnosticsOfCurrentFile(): FileDiagnostic | undefined { + if (vscode.window.activeTextEditor) { + return { + diagnostics: vscode.languages.getDiagnostics(vscode.window.activeTextEditor.document.uri), + filepath: vscode.window.activeTextEditor.document.uri.fsPath, + } + } + return undefined +} + +export type FileDiagnostic = { + filepath: string + diagnostics: vscode.Diagnostic[] +} + +export function getDiagnosticsDifferences( + oldDiagnostics: FileDiagnostic | undefined, + newDiagnostics: FileDiagnostic | undefined +): { added: vscode.Diagnostic[]; removed: vscode.Diagnostic[] } { + const result: { added: vscode.Diagnostic[]; removed: vscode.Diagnostic[] } = { added: [], removed: [] } + if ( + oldDiagnostics === undefined || + newDiagnostics === undefined || + newDiagnostics.filepath !== oldDiagnostics.filepath + ) { + return result + } + + // Create maps using diagnostic key for uniqueness + const oldMap = new Map(oldDiagnostics.diagnostics.map((d) => [getDiagnosticKey(d), d])) + const newMap = new Map(newDiagnostics.diagnostics.map((d) => [getDiagnosticKey(d), d])) + + // Get added diagnostics (in new but not in old) + result.added = [...newMap.values()].filter((d) => !oldMap.has(getDiagnosticKey(d))) + + // Get removed diagnostics (in old but not in new) + result.removed = [...oldMap.values()].filter((d) => !newMap.has(getDiagnosticKey(d))) + + return result +} + +export function toIdeDiagnostics(diagnostic: vscode.Diagnostic): IdeDiagnostic { + const severity = + diagnostic.severity === vscode.DiagnosticSeverity.Error + ? 'ERROR' + : diagnostic.severity === vscode.DiagnosticSeverity.Warning + ? 'WARNING' + : diagnostic.severity === vscode.DiagnosticSeverity.Hint + ? 'HINT' + : 'INFORMATION' + + return { + ideDiagnosticType: getDiagnosticsType(diagnostic.message), + severity: severity, + source: diagnostic.source, + range: { + start: { + line: diagnostic.range.start.line, + character: diagnostic.range.start.character, + }, + end: { + line: diagnostic.range.end.line, + character: diagnostic.range.end.character, + }, + }, + } +} + +export function getDiagnosticsType(message: string): string { + const errorTypes = new Map([ + ['SYNTAX_ERROR', ['expected', 'indent', 'syntax']], + ['TYPE_ERROR', ['type', 'cast']], + ['REFERENCE_ERROR', ['undefined', 'not defined', 'undeclared', 'reference']], + ['BEST_PRACTICE', ['deprecated', 'unused', 'uninitialized', 'not initialized']], + ['SECURITY', ['security', 'vulnerability']], + ]) + + const lowercaseMessage = message.toLowerCase() + + for (const [errorType, keywords] of errorTypes) { + if (keywords.some((keyword) => lowercaseMessage.includes(keyword))) { + return errorType + } + } + + return 'OTHER' +} + +/** + * Generates a unique MD5 hash key for a VS Code diagnostic object. + * + * @param diagnostic - A VS Code Diagnostic object containing information about a code diagnostic + * @returns A 32-character hexadecimal MD5 hash string that uniquely identifies the diagnostic + * + * @description + * Creates a deterministic hash by combining the diagnostic's message, severity, code, and source. + * This hash can be used as a unique identifier for deduplication or tracking purposes. + * Note: range is not in the hashed string because a diagnostic can move and its range can change within the editor + */ +function getDiagnosticKey(diagnostic: vscode.Diagnostic): string { + const jsonStr = JSON.stringify({ + message: diagnostic.message, + severity: diagnostic.severity, + code: diagnostic.code, + source: diagnostic.source, + }) + + return crypto.createHash('md5').update(jsonStr).digest('hex') +} diff --git a/packages/core/src/codewhisperer/util/telemetryHelper.ts b/packages/core/src/codewhisperer/util/telemetryHelper.ts index 4bb3b92dc33..060a5ecb282 100644 --- a/packages/core/src/codewhisperer/util/telemetryHelper.ts +++ b/packages/core/src/codewhisperer/util/telemetryHelper.ts @@ -26,9 +26,12 @@ import { getLogger } from '../../shared/logger/logger' import { session } from './codeWhispererSession' import { CodeWhispererSupplementalContext } from '../models/model' import { FeatureConfigProvider } from '../../shared/featureConfig' -import { CodeScanRemediationsEventType } from '../client/codewhispereruserclient' +import CodeWhispererUserClient, { CodeScanRemediationsEventType } from '../client/codewhispereruserclient' import { CodeAnalysisScope as CodeAnalysisScopeClientSide } from '../models/constants' import { Session } from '../../amazonqTest/chat/session/session' +import { sleep } from '../../shared/utilities/timeoutUtils' +import { getDiagnosticsDifferences, getDiagnosticsOfCurrentFile, toIdeDiagnostics } from './diagnosticsUtil' +import { Auth } from '../../auth/auth' export class TelemetryHelper { // Some variables for client component latency @@ -422,46 +425,56 @@ export class TelemetryHelper { e2eLatency = 0.0 } - client - .sendTelemetryEvent({ - telemetryEvent: { - userTriggerDecisionEvent: { - sessionId: sessionId, - requestId: this.sessionDecisions[0].codewhispererFirstRequestId, - customizationArn: selectedCustomization.arn === '' ? undefined : selectedCustomization.arn, - programmingLanguage: { - languageName: runtimeLanguageContext.toRuntimeLanguage( - this.sessionDecisions[0].codewhispererLanguage - ), - }, - completionType: this.getSendTelemetryCompletionType(aggregatedCompletionType), - suggestionState: this.getSendTelemetrySuggestionState(aggregatedSuggestionState), - recommendationLatencyMilliseconds: e2eLatency, - triggerToResponseLatencyMilliseconds: session.timeToFirstRecommendation, - perceivedLatencyMilliseconds: session.perceivedLatency, - timestamp: new Date(Date.now()), - suggestionReferenceCount: referenceCount, - generatedLine: generatedLines, - numberOfRecommendations: suggestionCount, - acceptedCharacterCount: acceptedRecommendationContent.length, - }, - }, - profileArn: profile?.arn, - }) - .then() - .catch((error) => { - let requestId: string | undefined - if (isAwsError(error)) { - requestId = error.requestId - } + const userTriggerDecisionEvent: CodeWhispererUserClient.UserTriggerDecisionEvent = { + sessionId: sessionId, + requestId: this.sessionDecisions[0].codewhispererFirstRequestId, + customizationArn: selectedCustomization.arn === '' ? undefined : selectedCustomization.arn, + programmingLanguage: { + languageName: runtimeLanguageContext.toRuntimeLanguage(this.sessionDecisions[0].codewhispererLanguage), + }, + completionType: this.getSendTelemetryCompletionType(aggregatedCompletionType), + suggestionState: this.getSendTelemetrySuggestionState(aggregatedSuggestionState), + recommendationLatencyMilliseconds: e2eLatency, + triggerToResponseLatencyMilliseconds: session.timeToFirstRecommendation, + perceivedLatencyMilliseconds: session.perceivedLatency, + timestamp: new Date(Date.now()), + suggestionReferenceCount: referenceCount, + generatedLine: generatedLines, + numberOfRecommendations: suggestionCount, + acceptedCharacterCount: acceptedRecommendationContent.length, + } + this.resetUserTriggerDecisionTelemetry() - getLogger().debug( - `Failed to sendTelemetryEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${ - error.message - }` + const sendEvent = () => + client + .sendTelemetryEvent({ + telemetryEvent: { userTriggerDecisionEvent: userTriggerDecisionEvent }, + profileArn: profile?.arn, + }) + .catch((error) => { + const requestId = isAwsError(error) ? error.requestId : undefined + getLogger().debug( + `Failed to sendTelemetryEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${error.message}` + ) + }) + + if (userTriggerDecisionEvent.suggestionState === 'ACCEPT' && Auth.instance.isInternalAmazonUser()) { + // wait 1 seconds for the user installed 3rd party LSP + // to update its diagnostics. + void sleep(1000).then(() => { + const diagnosticDiff = getDiagnosticsDifferences( + session.diagnosticsBeforeAccept, + getDiagnosticsOfCurrentFile() + ) + userTriggerDecisionEvent.addedIdeDiagnostics = diagnosticDiff.added.map((it) => toIdeDiagnostics(it)) + userTriggerDecisionEvent.removedIdeDiagnostics = diagnosticDiff.removed.map((it) => + toIdeDiagnostics(it) ) + void sendEvent() }) - this.resetUserTriggerDecisionTelemetry() + } else { + void sendEvent() + } } public getLastTriggerDecisionForClassifier() { diff --git a/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts b/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts index f2c447500da..2d9e01db9a0 100644 --- a/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts +++ b/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts @@ -36,7 +36,9 @@ import globals from '../../../shared/extensionGlobals' import { getLogger } from '../../../shared/logger/logger' import { codeWhispererClient } from '../../../codewhisperer/client/codewhisperer' import { isAwsError } from '../../../shared/errors' -import { ChatMessageInteractionType } from '../../../codewhisperer/client/codewhispereruserclient' +import CodeWhispererUserClient, { + ChatMessageInteractionType, +} from '../../../codewhisperer/client/codewhispereruserclient' import { supportedLanguagesList } from '../chat/chatRequest/converter' import { AuthUtil } from '../../../codewhisperer/util/authUtil' import { getSelectedCustomization } from '../../../codewhisperer/util/customizationUtil' @@ -44,6 +46,14 @@ import { undefinedIfEmpty } from '../../../shared/utilities/textUtilities' import { AdditionalContextPrompt } from '../../../amazonq/lsp/types' import { getUserPromptsDirectory, promptFileExtension } from '../../constants' import { isInDirectory } from '../../../shared/filesystemUtilities' +import { sleep } from '../../../shared/utilities/timeoutUtils' +import { + FileDiagnostic, + getDiagnosticsDifferences, + getDiagnosticsOfCurrentFile, + toIdeDiagnostics, +} from '../../../codewhisperer/util/diagnosticsUtil' +import { Auth } from '../../../auth/auth' export function logSendTelemetryEventFailure(error: any) { let requestId: string | undefined @@ -92,11 +102,17 @@ export class CWCTelemetryHelper { } > = new Map() + private documentDiagnostics: FileDiagnostic | undefined = undefined + constructor(sessionStorage: ChatSessionStorage, triggerEventsStorage: TriggerEventsStorage) { this.sessionStorage = sessionStorage this.triggerEventsStorage = triggerEventsStorage } + public setDocumentDiagnostics() { + this.documentDiagnostics = getDiagnosticsOfCurrentFile() + } + public static init(sessionStorage: ChatSessionStorage, triggerEventsStorage: TriggerEventsStorage) { const lastInstance = CWCTelemetryHelper.instance if (lastInstance !== undefined) { @@ -359,26 +375,50 @@ export class CWCTelemetryHelper { } telemetry.amazonq_interactWithMessage.emit({ ...event, ...additionalContextInfo }) - codeWhispererClient - .sendTelemetryEvent({ - telemetryEvent: { - chatInteractWithMessageEvent: { - conversationId: event.cwsprChatConversationId, - messageId: event.cwsprChatMessageId, - interactionType: this.getCWClientTelemetryInteractionType(event.cwsprChatInteractionType), - interactionTarget: event.cwsprChatInteractionTarget, - acceptedCharacterCount: event.cwsprChatAcceptedCharactersLength, - acceptedLineCount: event.cwsprChatAcceptedNumberOfLines, - acceptedSnippetHasReference: false, - hasProjectLevelContext: this.responseWithContextInfo.get(event.cwsprChatMessageId) - ?.cwsprChatHasProjectContext, - customizationArn: undefinedIfEmpty(getSelectedCustomization().arn), - }, - }, - profileArn: AuthUtil.instance.regionProfileManager.activeRegionProfile?.arn, + const interactWithMessageEvent: CodeWhispererUserClient.ChatInteractWithMessageEvent = { + conversationId: event.cwsprChatConversationId, + messageId: event.cwsprChatMessageId, + interactionType: this.getCWClientTelemetryInteractionType(event.cwsprChatInteractionType), + interactionTarget: event.cwsprChatInteractionTarget, + acceptedCharacterCount: event.cwsprChatAcceptedCharactersLength, + acceptedLineCount: event.cwsprChatAcceptedNumberOfLines, + acceptedSnippetHasReference: false, + hasProjectLevelContext: this.responseWithContextInfo.get(event.cwsprChatMessageId) + ?.cwsprChatHasProjectContext, + customizationArn: undefinedIfEmpty(getSelectedCustomization().arn), + } + if (interactWithMessageEvent.interactionType === 'INSERT_AT_CURSOR' && Auth.instance.isInternalAmazonUser()) { + // wait 1 seconds for the user installed 3rd party LSP + // to update its diagnostics. + void sleep(1000).then(() => { + const diagnosticDiff = getDiagnosticsDifferences( + this.documentDiagnostics, + getDiagnosticsOfCurrentFile() + ) + interactWithMessageEvent.addedIdeDiagnostics = diagnosticDiff.added.map((it) => toIdeDiagnostics(it)) + interactWithMessageEvent.removedIdeDiagnostics = diagnosticDiff.removed.map((it) => + toIdeDiagnostics(it) + ) + codeWhispererClient + .sendTelemetryEvent({ + telemetryEvent: { + chatInteractWithMessageEvent: interactWithMessageEvent, + }, + }) + .then() + .catch(logSendTelemetryEventFailure) }) - .then() - .catch(logSendTelemetryEventFailure) + } else { + codeWhispererClient + .sendTelemetryEvent({ + telemetryEvent: { + chatInteractWithMessageEvent: interactWithMessageEvent, + }, + profileArn: AuthUtil.instance.regionProfileManager.activeRegionProfile?.arn, + }) + .then() + .catch(logSendTelemetryEventFailure) + } } private getCWClientTelemetryInteractionType(type: CwsprChatInteractionType): ChatMessageInteractionType { diff --git a/packages/core/src/test/amazonq/common/contentController.test.ts b/packages/core/src/test/amazonq/common/contentController.test.ts index f0fb069844d..aac3b852e96 100644 --- a/packages/core/src/test/amazonq/common/contentController.test.ts +++ b/packages/core/src/test/amazonq/common/contentController.test.ts @@ -6,12 +6,16 @@ import * as vscode from 'vscode' import assert from 'assert' import { EditorContentController } from '../../../amazonq/commons/controllers/contentController' import { toTextEditor } from '../../testUtil' +import { CWCTelemetryHelper } from '../../../codewhispererChat/controllers/chat/telemetryHelper' +import { ChatSessionStorage } from '../../../codewhispererChat/storages/chatSession' +import { TriggerEventsStorage } from '../../../codewhispererChat' describe('contentController', () => { let controller: EditorContentController beforeEach(async () => { controller = new EditorContentController() + CWCTelemetryHelper.instance = new CWCTelemetryHelper(new ChatSessionStorage(), new TriggerEventsStorage()) }) describe('insertTextAtCursorPosition', () => { @@ -31,7 +35,7 @@ describe('contentController', () => { } }) - it('insert code when left hand size has non empty character', async () => { + it('insert code when left hand size has non empty character 2', async () => { const editor = await toTextEditor('def hello_world():\n ', 'test.py') if (editor) { const pos = new vscode.Position(0, 4)