Skip to content

Commit 360c614

Browse files
siakmun-awssiakmun-aws
committed
Sending metric data for onCodeGeneration
1 parent 7789d96 commit 360c614

File tree

5 files changed

+190
-30
lines changed

5 files changed

+190
-30
lines changed

‎packages/core/src/amazonqFeatureDev/client/featureDev.ts‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,12 @@ import { createCodeWhispererChatStreamingClient } from '../../shared/clients/cod
2525
import { getClientId, getOptOutPreference, getOperatingSystem } from '../../shared/telemetry/util'
2626
import { extensionVersion } from '../../shared/vscode/env'
2727
import apiConfig = require('./codewhispererruntime-2022-11-11.json')
28-
import { FeatureDevCodeAcceptanceEvent, FeatureDevCodeGenerationEvent, TelemetryEvent } from './featuredevproxyclient'
28+
import {
29+
FeatureDevCodeAcceptanceEvent,
30+
FeatureDevCodeGenerationEvent,
31+
MetricData,
32+
TelemetryEvent,
33+
} from './featuredevproxyclient'
2934

3035
// Re-enable once BE is able to handle retries.
3136
const writeAPIRetryOptions = {
@@ -299,6 +304,32 @@ export class FeatureDevClient {
299304
await this.sendFeatureDevEvent('featureDevCodeAcceptanceEvent', event)
300305
}
301306

307+
public async sendMetricDataRaw(event: MetricData) {
308+
getLogger().debug(
309+
`featureDevCodeGenerationMetricData: metricName: ${event.metricName} metricValue: ${event.metricValue} dimensions: ${event.dimensions}`
310+
)
311+
await this.sendFeatureDevEvent('metricData', event)
312+
}
313+
314+
public async sendMetricData(operationName: string, result: string) {
315+
await this.sendMetricDataRaw({
316+
metricName: 'Operation',
317+
metricValue: 1,
318+
timestamp: new Date(Date.now()),
319+
product: 'Amazon Q For VS Code',
320+
dimensions: [
321+
{
322+
name: 'operationName',
323+
value: operationName,
324+
},
325+
{
326+
name: 'result',
327+
value: result,
328+
},
329+
],
330+
})
331+
}
332+
302333
public async sendFeatureDevEvent<T extends keyof TelemetryEvent>(
303334
eventName: T,
304335
event: NonNullable<TelemetryEvent[T]>

‎packages/core/src/amazonqFeatureDev/controllers/chat/controller.ts‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import {
3030
import { codeGenRetryLimit, defaultRetryLimit } from '../../limits'
3131
import { Session } from '../../session/session'
3232
import { featureDevScheme, featureName } from '../../constants'
33-
import { DeletedFileInfo, DevPhase, type NewFileInfo } from '../../types'
33+
import { DeletedFileInfo, DevPhase, MetricDataOperationName, MetricDataResult, type NewFileInfo } from '../../types'
3434
import { AuthUtil } from '../../../codewhisperer/util/authUtil'
3535
import { AuthController } from '../../../amazonq/auth/controller'
3636
import { getLogger } from '../../../shared/logger'
@@ -413,6 +413,10 @@ export class FeatureDevController {
413413
canBeVoted: true,
414414
})
415415
this.messenger.sendUpdatePlaceholder(tabID, i18n('AWS.amazonq.featureDev.pillText.generatingCode'))
416+
await session.sendMetricDataTelemetry(
417+
MetricDataOperationName.START_CODE_GENERATION,
418+
MetricDataResult.SUCCESS
419+
)
416420
await session.send(message)
417421
const filePaths = session.state.filePaths ?? []
418422
const deletedFiles = session.state.deletedFiles ?? []
@@ -484,8 +488,45 @@ export class FeatureDevController {
484488
})
485489
await session.updateChatAnswer(tabID, i18n('AWS.amazonq.featureDev.pillText.acceptAllChanges'))
486490
await session.sendLinesOfCodeGeneratedTelemetry()
491+
await session.sendMetricDataTelemetry(
492+
MetricDataOperationName.END_CODE_GENERATION,
493+
MetricDataResult.SUCCESS
494+
)
487495
}
488496
this.messenger.sendUpdatePlaceholder(tabID, i18n('AWS.amazonq.featureDev.pillText.selectOption'))
497+
} catch (err: any) {
498+
getLogger().error(`${featureName}: Error during code generation: ${err}`)
499+
500+
switch (err.constructor.name) {
501+
case FeatureDevServiceError.name:
502+
if (err.code === 'EmptyPatchException') {
503+
await session.sendMetricDataTelemetry(
504+
MetricDataOperationName.END_CODE_GENERATION,
505+
MetricDataResult.LLMFAILURE
506+
)
507+
} else {
508+
await session.sendMetricDataTelemetry(
509+
MetricDataOperationName.END_CODE_GENERATION,
510+
MetricDataResult.ERROR
511+
)
512+
}
513+
break
514+
case PromptRefusalException.name:
515+
case NoChangeRequiredException.name:
516+
await session.sendMetricDataTelemetry(
517+
MetricDataOperationName.END_CODE_GENERATION,
518+
MetricDataResult.ERROR
519+
)
520+
break
521+
default:
522+
await session.sendMetricDataTelemetry(
523+
MetricDataOperationName.END_CODE_GENERATION,
524+
MetricDataResult.FAULT
525+
)
526+
527+
break
528+
}
529+
throw err
489530
} finally {
490531
// Finish processing the event
491532

‎packages/core/src/amazonqFeatureDev/session/session.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,10 @@ export class Session {
285285
return { leftPath, rightPath, ...diff }
286286
}
287287

288+
public async sendMetricDataTelemetry(operationName: string, result: string) {
289+
await this.proxyClient.sendMetricData(operationName, result)
290+
}
291+
288292
public async sendLinesOfCodeGeneratedTelemetry() {
289293
let charactersOfCodeGenerated = 0
290294
let linesOfCodeGenerated = 0

‎packages/core/src/amazonqFeatureDev/types.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,15 @@ export interface UpdateFilesPathsParams {
115115
messageId: string
116116
disableFileActions?: boolean
117117
}
118+
119+
export enum MetricDataOperationName {
120+
START_CODE_GENERATION = 'StartCodeGeneration',
121+
END_CODE_GENERATION = 'EndCodeGeneration',
122+
}
123+
124+
export enum MetricDataResult {
125+
SUCCESS = 'Success',
126+
FAULT = 'Fault',
127+
ERROR = 'Error',
128+
LLMFAILURE = 'LLMFailure',
129+
}

‎packages/core/src/test/amazonqFeatureDev/controllers/chat/controller.test.ts‎

Lines changed: 100 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,13 @@ import * as path from 'path'
99
import sinon from 'sinon'
1010
import { waitUntil } from '../../../../shared/utilities/timeoutUtils'
1111
import { ControllerSetup, createController, createSession, generateVirtualMemoryUri } from '../../utils'
12-
import { CurrentWsFolders, DeletedFileInfo, NewFileInfo } from '../../../../amazonqFeatureDev/types'
12+
import {
13+
CurrentWsFolders,
14+
DeletedFileInfo,
15+
MetricDataOperationName,
16+
MetricDataResult,
17+
NewFileInfo,
18+
} from '../../../../amazonqFeatureDev/types'
1319
import { Session } from '../../../../amazonqFeatureDev/session/session'
1420
import { Prompter } from '../../../../shared/ui/prompter'
1521
import { assertTelemetry, toFile } from '../../../testUtil'
@@ -36,6 +42,7 @@ import { AuthUtil } from '../../../../codewhisperer'
3642
import { featureDevScheme, featureName, messageWithConversationId } from '../../../../amazonqFeatureDev'
3743
import { i18n } from '../../../../shared/i18n-helper'
3844
import { FollowUpTypes } from '../../../../amazonq/commons/types'
45+
import { ToolkitError } from '../../../../shared'
3946

4047
let mockGetCodeGeneration: sinon.SinonStub
4148
describe('Controller', () => {
@@ -395,6 +402,46 @@ describe('Controller', () => {
395402
})
396403

397404
describe('processUserChatMessage', function () {
405+
// TODO: fix disablePreviousFileList error
406+
const runs = [
407+
{ name: 'ContentLengthError', error: new ContentLengthError() },
408+
{
409+
name: 'MonthlyConversationLimitError',
410+
error: new MonthlyConversationLimitError('Service Quota Exceeded'),
411+
},
412+
{
413+
name: 'FeatureDevServiceErrorGuardrailsException',
414+
error: new FeatureDevServiceError(
415+
i18n('AWS.amazonq.featureDev.error.codeGen.default'),
416+
'GuardrailsException'
417+
),
418+
},
419+
{
420+
name: 'FeatureDevServiceErrorEmptyPatchException',
421+
error: new FeatureDevServiceError(
422+
i18n('AWS.amazonq.featureDev.error.throttling'),
423+
'EmptyPatchException'
424+
),
425+
},
426+
{
427+
name: 'FeatureDevServiceErrorThrottlingException',
428+
error: new FeatureDevServiceError(
429+
i18n('AWS.amazonq.featureDev.error.codeGen.default'),
430+
'ThrottlingException'
431+
),
432+
},
433+
{ name: 'UploadCodeError', error: new UploadCodeError('403: Forbiden') },
434+
{ name: 'UserMessageNotFoundError', error: new UserMessageNotFoundError() },
435+
{ name: 'TabIdNotFoundError', error: new TabIdNotFoundError() },
436+
{ name: 'PrepareRepoFailedError', error: new PrepareRepoFailedError() },
437+
{ name: 'PromptRefusalException', error: new PromptRefusalException() },
438+
{ name: 'ZipFileError', error: new ZipFileError() },
439+
{ name: 'CodeIterationLimitError', error: new CodeIterationLimitError() },
440+
{ name: 'UploadURLExpired', error: new UploadURLExpired() },
441+
{ name: 'NoChangeRequiredException', error: new NoChangeRequiredException() },
442+
{ name: 'default', error: new ToolkitError('Default', { code: 'Default' }) },
443+
]
444+
398445
async function fireChatMessage() {
399446
const getSessionStub = sinon.stub(controllerSetup.sessionStorage, 'getSession').resolves(session)
400447

@@ -410,38 +457,63 @@ describe('Controller', () => {
410457
}, {})
411458
}
412459

413-
describe('processErrorChatMessage', function () {
414-
// TODO: fix disablePreviousFileList error
415-
const runs = [
416-
{ name: 'ContentLengthError', error: new ContentLengthError() },
417-
{
418-
name: 'MonthlyConversationLimitError',
419-
error: new MonthlyConversationLimitError('Service Quota Exceeded'),
420-
},
421-
{
422-
name: 'FeatureDevServiceError',
423-
error: new FeatureDevServiceError(
424-
i18n('AWS.amazonq.featureDev.error.codeGen.default'),
425-
'GuardrailsException'
426-
),
427-
},
428-
{ name: 'UploadCodeError', error: new UploadCodeError('403: Forbiden') },
429-
{ name: 'UserMessageNotFoundError', error: new UserMessageNotFoundError() },
430-
{ name: 'TabIdNotFoundError', error: new TabIdNotFoundError() },
431-
{ name: 'PrepareRepoFailedError', error: new PrepareRepoFailedError() },
432-
{ name: 'PromptRefusalException', error: new PromptRefusalException() },
433-
{ name: 'ZipFileError', error: new ZipFileError() },
434-
{ name: 'CodeIterationLimitError', error: new CodeIterationLimitError() },
435-
{ name: 'UploadURLExpired', error: new UploadURLExpired() },
436-
{ name: 'NoChangeRequiredException', error: new NoChangeRequiredException() },
437-
{ name: 'default', error: new Error() },
438-
]
460+
describe('onCodeGeneration', function () {
461+
async function verifyException(error: ToolkitError) {
462+
sinon.stub(session, 'preloader').resolves()
463+
sinon.stub(session, 'send').throws(error)
464+
const sendMetricDataTelemetrySpy = sinon.stub(session, 'sendMetricDataTelemetry')
465+
466+
await fireChatMessage()
467+
468+
sendMetricDataTelemetrySpy.calledWith(
469+
MetricDataOperationName.START_CODE_GENERATION,
470+
MetricDataResult.SUCCESS
471+
)
439472

473+
switch (error.constructor.name) {
474+
case FeatureDevServiceError.name:
475+
if (error.code === 'EmptyPatchException') {
476+
sendMetricDataTelemetrySpy.calledWith(
477+
MetricDataOperationName.END_CODE_GENERATION,
478+
MetricDataResult.LLMFAILURE
479+
)
480+
} else {
481+
sendMetricDataTelemetrySpy.calledWith(
482+
MetricDataOperationName.END_CODE_GENERATION,
483+
MetricDataResult.ERROR
484+
)
485+
}
486+
break
487+
case PromptRefusalException.name:
488+
case NoChangeRequiredException.name:
489+
sendMetricDataTelemetrySpy.calledWith(
490+
MetricDataOperationName.END_CODE_GENERATION,
491+
MetricDataResult.ERROR
492+
)
493+
break
494+
default:
495+
sendMetricDataTelemetrySpy.calledWith(
496+
MetricDataOperationName.END_CODE_GENERATION,
497+
MetricDataResult.FAULT
498+
)
499+
500+
break
501+
}
502+
}
503+
504+
runs.forEach((run) => {
505+
it(`sends operation telemetry on ${run.name}`, async function () {
506+
await verifyException(run.error)
507+
})
508+
})
509+
})
510+
511+
describe('processErrorChatMessage', function () {
440512
function createTestErrorMessage(message: string) {
441513
return createUserFacingErrorMessage(`${featureName} request failed: ${message}`)
442514
}
443515

444-
async function verifyException(error: Error) {
516+
async function verifyException(error: ToolkitError) {
445517
sinon.stub(session, 'preloader').throws(error)
446518
const sendAnswerSpy = sinon.stub(controllerSetup.messenger, 'sendAnswer')
447519
const sendErrorMessageSpy = sinon.stub(controllerSetup.messenger, 'sendErrorMessage')

0 commit comments

Comments
 (0)