diff --git a/packages/amazonq/test/unit/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts b/packages/amazonq/test/unit/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts index 7fe3b3b7840..ee001b3328d 100644 --- a/packages/amazonq/test/unit/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts +++ b/packages/amazonq/test/unit/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts @@ -6,7 +6,13 @@ import assert from 'assert' import * as sinon from 'sinon' import * as vscode from 'vscode' -import { CodeWhispererCodeCoverageTracker, vsCodeState, TelemetryHelper, AuthUtil } from 'aws-core-vscode/codewhisperer' +import { + CodeWhispererCodeCoverageTracker, + vsCodeState, + TelemetryHelper, + AuthUtil, + getUnmodifiedAcceptedTokens, +} from 'aws-core-vscode/codewhisperer' import { createMockDocument, createMockTextEditor, resetCodeWhispererGlobalVariables } from 'aws-core-vscode/test' import { globals } from 'aws-core-vscode/shared' import { assertTelemetryCurried } from 'aws-core-vscode/test' @@ -150,14 +156,13 @@ describe('codewhispererCodecoverageTracker', function () { }) it('Should return correct unmodified accepted tokens count', function () { - const tracker = CodeWhispererCodeCoverageTracker.getTracker(language) - assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('foo', 'fou'), 2) - assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('foo', 'f11111oo'), 3) - assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('foo', 'fo'), 2) - assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('helloworld', 'HelloWorld'), 8) - assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('helloworld', 'World'), 4) - assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('CodeWhisperer', 'CODE'), 1) - assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('CodeWhisperer', 'CodeWhispererGood'), 13) + assert.strictEqual(getUnmodifiedAcceptedTokens('foo', 'fou'), 2) + assert.strictEqual(getUnmodifiedAcceptedTokens('foo', 'f11111oo'), 3) + assert.strictEqual(getUnmodifiedAcceptedTokens('foo', 'fo'), 2) + assert.strictEqual(getUnmodifiedAcceptedTokens('helloworld', 'HelloWorld'), 8) + assert.strictEqual(getUnmodifiedAcceptedTokens('helloworld', 'World'), 4) + assert.strictEqual(getUnmodifiedAcceptedTokens('CodeWhisperer', 'CODE'), 1) + assert.strictEqual(getUnmodifiedAcceptedTokens('CodeWhisperer', 'CodeWhispererGood'), 13) }) }) diff --git a/packages/core/src/codewhisperer/client/user-service-2.json b/packages/core/src/codewhisperer/client/user-service-2.json index bda8d16922d..3a94931dddb 100644 --- a/packages/core/src/codewhisperer/client/user-service-2.json +++ b/packages/core/src/codewhisperer/client/user-service-2.json @@ -2197,14 +2197,24 @@ }, "UserModificationEvent": { "type": "structure", - "required": ["sessionId", "requestId", "programmingLanguage", "modificationPercentage", "timestamp"], + "required": [ + "sessionId", + "requestId", + "programmingLanguage", + "modificationPercentage", + "timestamp", + "acceptedCharacterCount", + "unmodifiedAcceptedCharacterCount" + ], "members": { "sessionId": { "shape": "UUID" }, "requestId": { "shape": "UUID" }, "programmingLanguage": { "shape": "ProgrammingLanguage" }, "modificationPercentage": { "shape": "Double" }, "customizationArn": { "shape": "CustomizationArn" }, - "timestamp": { "shape": "Timestamp" } + "timestamp": { "shape": "Timestamp" }, + "acceptedCharacterCount": { "shape": "PrimitiveInteger" }, + "unmodifiedAcceptedCharacterCount": { "shape": "PrimitiveInteger" } } }, "UserTriggerDecisionEvent": { @@ -2230,7 +2240,9 @@ "triggerToResponseLatencyMilliseconds": { "shape": "Double" }, "suggestionReferenceCount": { "shape": "PrimitiveInteger" }, "generatedLine": { "shape": "PrimitiveInteger" }, - "numberOfRecommendations": { "shape": "PrimitiveInteger" } + "numberOfRecommendations": { "shape": "PrimitiveInteger" }, + "perceivedLatencyMilliseconds": { "shape": "Double" }, + "acceptedCharacterCount": { "shape": "PrimitiveInteger" } } }, "ValidationException": { diff --git a/packages/core/src/codewhisperer/service/inlineCompletionService.ts b/packages/core/src/codewhisperer/service/inlineCompletionService.ts index 69bd9bdb887..715fd93ad2d 100644 --- a/packages/core/src/codewhisperer/service/inlineCompletionService.ts +++ b/packages/core/src/codewhisperer/service/inlineCompletionService.ts @@ -112,7 +112,6 @@ export class InlineCompletionService { await this.setState('loading') - TelemetryHelper.instance.setInvocationStartTime(performance.now()) RecommendationHandler.instance.checkAndResetCancellationTokens() RecommendationHandler.instance.documentUri = editor.document.uri let response: GetRecommendationsResponse = { diff --git a/packages/core/src/codewhisperer/service/recommendationHandler.ts b/packages/core/src/codewhisperer/service/recommendationHandler.ts index 60e877064bb..1fd46541c11 100644 --- a/packages/core/src/codewhisperer/service/recommendationHandler.ts +++ b/packages/core/src/codewhisperer/service/recommendationHandler.ts @@ -256,7 +256,7 @@ export class RecommendationHandler { sessionId = resp?.$response?.httpResponse?.headers['x-amzn-sessionid'] TelemetryHelper.instance.setFirstResponseRequestId(requestId) if (page === 0) { - TelemetryHelper.instance.setTimeToFirstRecommendation(performance.now()) + session.setTimeToFirstRecommendation(performance.now()) } if (nextToken === '') { TelemetryHelper.instance.setAllPaginationEndTime() diff --git a/packages/core/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts b/packages/core/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts index 2ec761ceb8c..925609ce185 100644 --- a/packages/core/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts +++ b/packages/core/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts @@ -8,7 +8,6 @@ import { getLogger } from '../../shared/logger/logger' import * as CodeWhispererConstants from '../models/constants' import globals from '../../shared/extensionGlobals' import { vsCodeState } from '../models/model' -import { distance } from 'fastest-levenshtein' import { CodewhispererLanguage, telemetry } from '../../shared/telemetry/telemetry' import { runtimeLanguageContext } from '../util/runtimeLanguageContext' import { TelemetryHelper } from '../util/telemetryHelper' @@ -16,6 +15,7 @@ import { AuthUtil } from '../util/authUtil' import { getSelectedCustomization } from '../util/customizationUtil' import { codeWhispererClient as client } from '../client/codewhisperer' import { isAwsError } from '../../shared/errors' +import { getUnmodifiedAcceptedTokens } from '../util/commonUtil' interface CodeWhispererToken { range: vscode.Range @@ -86,18 +86,10 @@ export class CodeWhispererCodeCoverageTracker { for (let i = 0; i < this._acceptedTokens[filename].length; i++) { const oldText = this._acceptedTokens[filename][i].text const newText = editor.document.getText(this._acceptedTokens[filename][i].range) - this._acceptedTokens[filename][i].accepted = this.getUnmodifiedAcceptedTokens(oldText, newText) + this._acceptedTokens[filename][i].accepted = getUnmodifiedAcceptedTokens(oldText, newText) } } } - // With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace), - // and thus the unmodified part of recommendation length can be deducted/approximated - // ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3 - // ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8 - // ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1 - public getUnmodifiedAcceptedTokens(origin: string, after: string) { - return Math.max(origin.length, after.length) - distance(origin, after) - } public emitCodeWhispererCodeContribution() { let totalTokens = 0 diff --git a/packages/core/src/codewhisperer/tracker/codewhispererTracker.ts b/packages/core/src/codewhisperer/tracker/codewhispererTracker.ts index 3322fa29990..05a6d83f3f0 100644 --- a/packages/core/src/codewhisperer/tracker/codewhispererTracker.ts +++ b/packages/core/src/codewhisperer/tracker/codewhispererTracker.ts @@ -15,7 +15,8 @@ import { codeWhispererClient } from '../client/codewhisperer' import { logSendTelemetryEventFailure } from '../../codewhispererChat/controllers/chat/telemetryHelper' import { Timeout } from '../../shared/utilities/timeoutUtils' import { getSelectedCustomization } from '../util/customizationUtil' -import { undefinedIfEmpty } from '../../shared' +import { isAwsError, undefinedIfEmpty } from '../../shared' +import { getUnmodifiedAcceptedTokens } from '../util/commonUtil' /** * This singleton class is mainly used for calculating the percentage of user modification. @@ -89,19 +90,20 @@ export class CodeWhispererTracker { public async emitTelemetryOnSuggestion(suggestion: AcceptedSuggestionEntry | InsertedCode) { let percentage = 1.0 + let currString = '' + const customizationArn = undefinedIfEmpty(getSelectedCustomization().arn) try { if (suggestion.fileUrl?.scheme !== '') { const document = await vscode.workspace.openTextDocument(suggestion.fileUrl) if (document) { - const currString = document.getText( - new vscode.Range(suggestion.startPosition, suggestion.endPosition) - ) + currString = document.getText(new vscode.Range(suggestion.startPosition, suggestion.endPosition)) percentage = this.checkDiff(currString, suggestion.originalString) } } } catch (e) { getLogger().verbose(`Exception Thrown from CodeWhispererTracker: ${e}`) + return } finally { if ('conversationID' in suggestion) { const event: AmazonqModifyCode = { @@ -120,7 +122,7 @@ export class CodeWhispererTracker { conversationId: event.cwsprChatConversationId, messageId: event.cwsprChatMessageId, modificationPercentage: event.cwsprChatModificationPercentage, - customizationArn: undefinedIfEmpty(getSelectedCustomization().arn), + customizationArn: customizationArn, }, }, }) @@ -139,9 +141,39 @@ export class CodeWhispererTracker { codewhispererCharactersAccepted: suggestion.originalString.length, codewhispererCharactersModified: 0, // TODO: currently we don't have an accurate number for this field with existing implementation }) - // TODO: - // Temperary comment out user modification event, need further discussion on how to calculate this metric - // TelemetryHelper.instance.sendUserModificationEvent(suggestion, percentage) + + codeWhispererClient + .sendTelemetryEvent({ + telemetryEvent: { + userModificationEvent: { + sessionId: suggestion.sessionId, + requestId: suggestion.requestId, + programmingLanguage: { languageName: suggestion.language }, + // deprecated % value and should not be used by service side + modificationPercentage: percentage, + customizationArn: customizationArn, + timestamp: new Date(), + acceptedCharacterCount: suggestion.originalString.length, + unmodifiedAcceptedCharacterCount: getUnmodifiedAcceptedTokens( + suggestion.originalString, + currString + ), + }, + }, + }) + .then() + .catch((error) => { + let requestId: string | undefined + if (isAwsError(error)) { + requestId = error.requestId + } + + getLogger().debug( + `Failed to send UserModificationEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${ + error.message + }` + ) + }) } } } diff --git a/packages/core/src/codewhisperer/util/codeWhispererSession.ts b/packages/core/src/codewhisperer/util/codeWhispererSession.ts index d6c06ef5350..f57fddaecd0 100644 --- a/packages/core/src/codewhisperer/util/codeWhispererSession.ts +++ b/packages/core/src/codewhisperer/util/codeWhispererSession.ts @@ -12,7 +12,7 @@ import { } from '../../shared/telemetry/telemetry.gen' import { GenerateRecommendationsRequest, ListRecommendationsRequest, Recommendation } from '../client/codewhisperer' import { Position } from 'vscode' -import { CodeWhispererSupplementalContext } from '../models/model' +import { CodeWhispererSupplementalContext, vsCodeState } from '../models/model' class CodeWhispererSession { static #instance: CodeWhispererSession @@ -41,6 +41,8 @@ class CodeWhispererSession { fetchCredentialStartTime = 0 sdkApiCallStartTime = 0 invokeSuggestionStartTime = 0 + timeToFirstRecommendation = 0 + firstSuggestionShowTime = 0 public static get instance() { return (this.#instance ??= new CodeWhispererSession()) @@ -58,6 +60,12 @@ class CodeWhispererSession { } } + setTimeToFirstRecommendation(timeToFirstRecommendation: number) { + if (this.invokeSuggestionStartTime) { + this.timeToFirstRecommendation = timeToFirstRecommendation - this.invokeSuggestionStartTime + } + } + setSuggestionState(index: number, value: string) { this.suggestionStates.set(index, value) } @@ -75,6 +83,14 @@ class CodeWhispererSession { return this.completionTypes.get(index) || 'Line' } + getPerceivedLatency(triggerType: CodewhispererTriggerType) { + if (triggerType === 'OnDemand') { + return this.timeToFirstRecommendation + } else { + return session.firstSuggestionShowTime - vsCodeState.lastUserModificationTime + } + } + reset() { this.sessionId = '' this.requestContext = { request: {} as any, supplementalMetadata: {} as any } diff --git a/packages/core/src/codewhisperer/util/commonUtil.ts b/packages/core/src/codewhisperer/util/commonUtil.ts index 201a5f1c595..1d624e77b5e 100644 --- a/packages/core/src/codewhisperer/util/commonUtil.ts +++ b/packages/core/src/codewhisperer/util/commonUtil.ts @@ -5,6 +5,7 @@ import * as vscode from 'vscode' import * as semver from 'semver' +import { distance } from 'fastest-levenshtein' import { isCloud9 } from '../../shared/extensionUtilities' import { getInlineSuggestEnabled } from '../../shared/utilities/editorUtilities' import { @@ -76,3 +77,12 @@ export function checkLeftContextKeywordsForJson(fileName: string, leftFileConten } return false } + +// With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace), +// and thus the unmodified part of recommendation length can be deducted/approximated +// ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3 +// ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8 +// ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1 +export function getUnmodifiedAcceptedTokens(origin: string, after: string) { + return Math.max(origin.length, after.length) - distance(origin, after) +} diff --git a/packages/core/src/codewhisperer/util/telemetryHelper.ts b/packages/core/src/codewhisperer/util/telemetryHelper.ts index 82f812ae1b7..3afb14f2c98 100644 --- a/packages/core/src/codewhisperer/util/telemetryHelper.ts +++ b/packages/core/src/codewhisperer/util/telemetryHelper.ts @@ -30,7 +30,6 @@ import { CodeScanRemediationsEventType } from '../client/codewhispereruserclient export class TelemetryHelper { // Some variables for client component latency private sdkApiCallEndTime = 0 - private firstSuggestionShowTime = 0 private allPaginationEndTime = 0 private firstResponseRequestId = '' // variables for user trigger decision @@ -41,8 +40,6 @@ export class TelemetryHelper { private typeAheadLength = 0 private timeSinceLastModification = 0 private lastTriggerDecisionTime = 0 - private invocationTime = 0 - private timeToFirstRecommendation = 0 private classifierResult?: number = undefined private classifierThreshold?: number = undefined // variables for tracking end to end sessions @@ -285,7 +282,7 @@ export class TelemetryHelper { codewhispererTimeSinceLastUserDecision: this.lastTriggerDecisionTime ? performance.now() - this.lastTriggerDecisionTime : undefined, - codewhispererTimeToFirstRecommendation: this.timeToFirstRecommendation, + codewhispererTimeToFirstRecommendation: session.timeToFirstRecommendation, codewhispererTriggerCharacter: autoTriggerType === 'SpecialCharacters' ? this.triggerChar : undefined, codewhispererSuggestionState: aggregatedSuggestionState, codewhispererPreviousSuggestionState: this.prevTriggerDecision, @@ -305,11 +302,11 @@ export class TelemetryHelper { this.prevTriggerDecision = this.getAggregatedSuggestionState(this.sessionDecisions) this.lastTriggerDecisionTime = performance.now() - // When we send a userTriggerDecision of Empty or Discard, we set the time users see the first - // suggestion to be now. - let e2eLatency = this.firstSuggestionShowTime - session.invokeSuggestionStartTime - if (e2eLatency < 0) { - e2eLatency = performance.now() - session.invokeSuggestionStartTime + // When we send a userTriggerDecision for neither Accept nor Reject, service side should not use this value + // and client side will set this value to 0.0. + let e2eLatency = session.firstSuggestionShowTime - session.invokeSuggestionStartTime + if (aggregatedSuggestionState !== 'Reject' && aggregatedSuggestionState !== 'Accept') { + e2eLatency = 0.0 } client @@ -327,8 +324,11 @@ export class TelemetryHelper { completionType: this.getSendTelemetryCompletionType(aggregatedCompletionType), suggestionState: this.getSendTelemetrySuggestionState(aggregatedSuggestionState), recommendationLatencyMilliseconds: e2eLatency, + triggerToResponseLatencyMilliseconds: session.timeToFirstRecommendation, + perceivedLatencyMilliseconds: session.getPerceivedLatency( + this.sessionDecisions[0].codewhispererTriggerType + ), timestamp: new Date(Date.now()), - triggerToResponseLatencyMilliseconds: this.timeToFirstRecommendation, suggestionReferenceCount: referenceCount, generatedLine: generatedLines, numberOfRecommendations: suggestionCount, @@ -377,16 +377,6 @@ export class TelemetryHelper { this.timeSinceLastModification = timeSinceLastModification } - public setInvocationStartTime(invocationTime: number) { - this.invocationTime = invocationTime - } - - public setTimeToFirstRecommendation(timeToFirstRecommendation: number) { - if (this.invocationTime) { - this.timeToFirstRecommendation = timeToFirstRecommendation - this.invocationTime - } - } - public setTraceId(traceId: string) { this.traceId = traceId } @@ -396,7 +386,7 @@ export class TelemetryHelper { this.triggerChar = '' this.typeAheadLength = 0 this.timeSinceLastModification = 0 - this.timeToFirstRecommendation = 0 + session.timeToFirstRecommendation = 0 this.classifierResult = undefined this.classifierThreshold = undefined } @@ -479,7 +469,7 @@ export class TelemetryHelper { session.sdkApiCallStartTime = 0 this.sdkApiCallEndTime = 0 session.fetchCredentialStartTime = 0 - this.firstSuggestionShowTime = 0 + session.firstSuggestionShowTime = 0 this.allPaginationEndTime = 0 this.firstResponseRequestId = '' } @@ -503,8 +493,8 @@ export class TelemetryHelper { } public setFirstSuggestionShowTime() { - if (this.firstSuggestionShowTime === 0 && this.sdkApiCallEndTime !== 0) { - this.firstSuggestionShowTime = performance.now() + if (session.firstSuggestionShowTime === 0 && this.sdkApiCallEndTime !== 0) { + session.firstSuggestionShowTime = performance.now() } } @@ -517,16 +507,16 @@ export class TelemetryHelper { // report client component latency after all pagination call finish // and at least one suggestion is shown to the user public tryRecordClientComponentLatency() { - if (this.firstSuggestionShowTime === 0 || this.allPaginationEndTime === 0) { + if (session.firstSuggestionShowTime === 0 || this.allPaginationEndTime === 0) { return } telemetry.codewhisperer_clientComponentLatency.emit({ codewhispererRequestId: this.firstResponseRequestId, codewhispererSessionId: session.sessionId, codewhispererFirstCompletionLatency: this.sdkApiCallEndTime - session.sdkApiCallStartTime, - codewhispererEndToEndLatency: this.firstSuggestionShowTime - session.invokeSuggestionStartTime, + codewhispererEndToEndLatency: session.firstSuggestionShowTime - session.invokeSuggestionStartTime, codewhispererAllCompletionsLatency: this.allPaginationEndTime - session.sdkApiCallStartTime, - codewhispererPostprocessingLatency: this.firstSuggestionShowTime - this.sdkApiCallEndTime, + codewhispererPostprocessingLatency: session.firstSuggestionShowTime - this.sdkApiCallEndTime, codewhispererCredentialFetchingLatency: session.sdkApiCallStartTime - session.fetchCredentialStartTime, codewhispererPreprocessingLatency: session.fetchCredentialStartTime - session.invokeSuggestionStartTime, codewhispererCompletionType: 'Line',