Skip to content

Commit 8ce5469

Browse files
KevinDing1KevinDing1
authored
telemetry(amazonq): /doc codegen #6822
This commit introduces metric telemetry for the /doc, which will be triggered during the code generation process. The new telemetry will: 1. Record successful and failure generation requests 2. Track failed requests with specific error types 3. Enable the setup of client monitoring alarms base on the metric data
1 parent 5768cf5 commit 8ce5469

File tree

7 files changed

+257
-51
lines changed

7 files changed

+257
-51
lines changed

‎packages/core/src/amazonqDoc/controllers/chat/controller.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import {
3030
SelectedFolderNotInWorkspaceFolderError,
3131
WorkspaceFolderNotFoundError,
3232
createUserFacingErrorMessage,
33+
getMetricResult,
3334
} from '../../../amazonqFeatureDev/errors'
3435
import { BaseChatSessionStorage } from '../../../amazonq/commons/baseChatStorage'
3536
import { DocMessenger } from '../../messenger'
@@ -44,8 +45,8 @@ import {
4445
import { getPathsFromZipFilePath, SvgFileExtension } from '../../../amazonq/util/files'
4546
import { FollowUpTypes } from '../../../amazonq/commons/types'
4647
import { DocGenerationTask, DocGenerationTasks } from '../docGenerationTask'
47-
import { DevPhase } from '../../types'
4848
import { normalize } from '../../../shared/utilities/pathUtils'
49+
import { DevPhase, MetricDataOperationName, MetricDataResult } from '../../types'
4950

5051
export interface ChatControllerEventEmitters {
5152
readonly processHumanChatMessage: EventEmitter<any>
@@ -557,6 +558,7 @@ export class DocController {
557558
await session.preloader(message)
558559

559560
try {
561+
await session.sendDocMetricData(MetricDataOperationName.StartDocGeneration, MetricDataResult.Success)
560562
await session.send(message, docGenerationTask.mode, docGenerationTask.folderPath)
561563
const filePaths = session.state.filePaths ?? []
562564
const deletedFiles = session.state.deletedFiles ?? []
@@ -627,6 +629,10 @@ export class DocController {
627629

628630
await session.sendDocTelemetryEvent(docGenerationEvent, 'generation')
629631
}
632+
} catch (err: any) {
633+
getLogger().error(`${featureName}: Error during doc generation: ${err}`)
634+
await session.sendDocMetricData(MetricDataOperationName.EndDocGeneration, getMetricResult(err))
635+
throw err
630636
} finally {
631637
if (session?.state?.tokenSource?.token.isCancellationRequested) {
632638
await this.newTask({ tabID })
@@ -636,6 +642,7 @@ export class DocController {
636642
this.messenger.sendChatInputEnabled(tabID, false)
637643
}
638644
}
645+
await session.sendDocMetricData(MetricDataOperationName.EndDocGeneration, MetricDataResult.Success)
639646
}
640647

641648
private authClicked(message: any) {

‎packages/core/src/amazonqDoc/errors.ts‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,60 +3,60 @@
33
* SPDX-License-Identifier: Apache-2.0
44
*/
55

6-
import { ToolkitError } from '../shared/errors'
6+
import { ClientError, ContentLengthError as CommonContentLengthError } from '../shared/errors'
77
import { i18n } from '../shared/i18n-helper'
88

9-
export class DocServiceError extends ToolkitError {
9+
export class DocClientError extends ClientError {
1010
remainingIterations?: number
1111
constructor(message: string, code: string, remainingIterations?: number) {
1212
super(message, { code })
1313
this.remainingIterations = remainingIterations
1414
}
1515
}
1616

17-
export class ReadmeTooLargeError extends DocServiceError {
17+
export class ReadmeTooLargeError extends DocClientError {
1818
constructor() {
1919
super(i18n('AWS.amazonq.doc.error.readmeTooLarge'), ReadmeTooLargeError.name)
2020
}
2121
}
2222

23-
export class ReadmeUpdateTooLargeError extends DocServiceError {
23+
export class ReadmeUpdateTooLargeError extends DocClientError {
2424
constructor(remainingIterations: number) {
2525
super(i18n('AWS.amazonq.doc.error.readmeUpdateTooLarge'), ReadmeUpdateTooLargeError.name, remainingIterations)
2626
}
2727
}
2828

29-
export class WorkspaceEmptyError extends DocServiceError {
29+
export class WorkspaceEmptyError extends DocClientError {
3030
constructor() {
3131
super(i18n('AWS.amazonq.doc.error.workspaceEmpty'), WorkspaceEmptyError.name)
3232
}
3333
}
3434

35-
export class NoChangeRequiredException extends DocServiceError {
35+
export class NoChangeRequiredException extends DocClientError {
3636
constructor() {
3737
super(i18n('AWS.amazonq.doc.error.noChangeRequiredException'), NoChangeRequiredException.name)
3838
}
3939
}
4040

41-
export class PromptRefusalException extends DocServiceError {
41+
export class PromptRefusalException extends DocClientError {
4242
constructor(remainingIterations: number) {
4343
super(i18n('AWS.amazonq.doc.error.promptRefusal'), PromptRefusalException.name, remainingIterations)
4444
}
4545
}
4646

47-
export class ContentLengthError extends DocServiceError {
47+
export class ContentLengthError extends CommonContentLengthError {
4848
constructor() {
49-
super(i18n('AWS.amazonq.doc.error.contentLengthError'), ContentLengthError.name)
49+
super(i18n('AWS.amazonq.doc.error.contentLengthError'), { code: ContentLengthError.name })
5050
}
5151
}
5252

53-
export class PromptTooVagueError extends DocServiceError {
53+
export class PromptTooVagueError extends DocClientError {
5454
constructor(remainingIterations: number) {
5555
super(i18n('AWS.amazonq.doc.error.promptTooVague'), PromptTooVagueError.name, remainingIterations)
5656
}
5757
}
5858

59-
export class PromptUnrelatedError extends DocServiceError {
59+
export class PromptUnrelatedError extends DocClientError {
6060
constructor(remainingIterations: number) {
6161
super(i18n('AWS.amazonq.doc.error.promptUnrelated'), PromptUnrelatedError.name, remainingIterations)
6262
}

‎packages/core/src/amazonqDoc/session/session.ts‎

Lines changed: 47 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,14 @@ import { FeatureDevClient } from '../../amazonqFeatureDev/client/featureDev'
1414
import { TelemetryHelper } from '../../amazonq/util/telemetryHelper'
1515
import { ConversationNotStartedState } from '../../amazonqFeatureDev/session/sessionState'
1616
import { logWithConversationId } from '../../amazonqFeatureDev/userFacingText'
17-
import { ConversationIdNotFoundError } from '../../amazonqFeatureDev/errors'
17+
import { ConversationIdNotFoundError, IllegalStateError } from '../../amazonqFeatureDev/errors'
1818
import { referenceLogText } from '../../amazonqFeatureDev/constants'
1919
import {
2020
DocInteractionType,
2121
DocV2AcceptanceEvent,
2222
DocV2GenerationEvent,
2323
SendTelemetryEventRequest,
24+
MetricData,
2425
} from '../../codewhisperer/client/codewhispereruserclient'
2526
import { getClientId, getOperatingSystem, getOptOutPreference } from '../../shared/telemetry/util'
2627
import { DocMessenger } from '../messenger'
@@ -72,7 +73,7 @@ export class Session {
7273

7374
get state() {
7475
if (!this._state) {
75-
throw new Error("State should be initialized before it's read")
76+
throw new IllegalStateError("State should be initialized before it's read")
7677
}
7778
return this._state
7879
}
@@ -269,15 +270,40 @@ export class Session {
269270
return { leftPath, rightPath, ...diff }
270271
}
271272

273+
public async sendDocMetricData(operationName: string, result: string) {
274+
const metricData = {
275+
metricName: 'Operation',
276+
metricValue: 1,
277+
timestamp: new Date(),
278+
product: 'DocGeneration',
279+
dimensions: [
280+
{
281+
name: 'operationName',
282+
value: operationName,
283+
},
284+
{
285+
name: 'result',
286+
value: result,
287+
},
288+
],
289+
}
290+
await this.sendDocTelemetryEvent(metricData, 'metric')
291+
}
292+
272293
public async sendDocTelemetryEvent(
273-
telemetryEvent: DocV2GenerationEvent | DocV2AcceptanceEvent,
274-
eventType: 'generation' | 'acceptance'
294+
telemetryEvent: DocV2GenerationEvent | DocV2AcceptanceEvent | MetricData,
295+
eventType: 'generation' | 'acceptance' | 'metric'
275296
) {
276297
const client = await this.proxyClient.getClient()
298+
const telemetryEventKey = {
299+
generation: 'docV2GenerationEvent',
300+
acceptance: 'docV2AcceptanceEvent',
301+
metric: 'metricData',
302+
}[eventType]
277303
try {
278304
const params: SendTelemetryEventRequest = {
279305
telemetryEvent: {
280-
[eventType === 'generation' ? 'docV2GenerationEvent' : 'docV2AcceptanceEvent']: telemetryEvent,
306+
[telemetryEventKey]: telemetryEvent,
281307
},
282308
optOutPreference: getOptOutPreference(),
283309
userContext: {
@@ -290,14 +316,23 @@ export class Session {
290316
}
291317

292318
const response = await client.sendTelemetryEvent(params).promise()
293-
getLogger().debug(
294-
`${featureName}: successfully sent docV2${eventType === 'generation' ? 'GenerationEvent' : 'AcceptanceEvent'}: ConversationId: ${telemetryEvent.conversationId} RequestId: ${response.$response.requestId}`
295-
)
319+
if (eventType === 'metric') {
320+
getLogger().debug(
321+
`${featureName}: successfully sent metricData: RequestId: ${response.$response.requestId}`
322+
)
323+
} else {
324+
getLogger().debug(
325+
`${featureName}: successfully sent docV2${eventType === 'generation' ? 'GenerationEvent' : 'AcceptanceEvent'}: ` +
326+
`ConversationId: ${(telemetryEvent as DocV2GenerationEvent | DocV2AcceptanceEvent).conversationId} ` +
327+
`RequestId: ${response.$response.requestId}`
328+
)
329+
}
296330
} catch (e) {
331+
const error = e as Error
332+
const eventTypeString = eventType === 'metric' ? 'metricData' : `doc ${eventType}`
297333
getLogger().error(
298-
`${featureName}: failed to send doc ${eventType} telemetry: ${(e as Error).name}: ${
299-
(e as Error).message
300-
} RequestId: ${(e as any).requestId}`
334+
`${featureName}: failed to send ${eventTypeString} telemetry: ${error.name}: ${error.message} ` +
335+
`RequestId: ${(e as any).$response?.requestId}`
301336
)
302337
}
303338
}
@@ -308,7 +343,7 @@ export class Session {
308343

309344
get uploadId() {
310345
if (!('uploadId' in this.state)) {
311-
throw new Error("UploadId has to be initialized before it's read")
346+
throw new IllegalStateError("UploadId has to be initialized before it's read")
312347
}
313348
return this.state.uploadId
314349
}

‎packages/core/src/amazonqDoc/session/sessionState.ts‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,13 @@
33
* SPDX-License-Identifier: Apache-2.0
44
*/
55

6-
import { ToolkitError } from '../../shared/errors'
76
import { DocGenerationStep, docScheme, getFileSummaryPercentage, Mode } from '../constants'
87

98
import { i18n } from '../../shared/i18n-helper'
109

1110
import { CurrentWsFolders, NewFileInfo, SessionState, SessionStateAction, SessionStateConfig } from '../types'
1211
import {
1312
ContentLengthError,
14-
DocServiceError,
1513
NoChangeRequiredException,
1614
PromptRefusalException,
1715
PromptTooVagueError,
@@ -20,11 +18,13 @@ import {
2018
ReadmeUpdateTooLargeError,
2119
WorkspaceEmptyError,
2220
} from '../errors'
21+
import { ApiClientError, ApiServiceError } from '../../amazonqFeatureDev/errors'
2322
import { DocMessenger } from '../messenger'
2423
import { BaseCodeGenState, BasePrepareCodeGenState, CreateNextStateParams } from '../../amazonq/session/sessionState'
2524
import { Intent } from '../../amazonq/commons/types'
2625
import { AmazonqCreateUpload, Span } from '../../shared/telemetry/telemetry'
2726
import { prepareRepoData, PrepareRepoDataOptions } from '../../amazonq/util/files'
27+
import { LlmError } from '../../amazonq/errors'
2828

2929
export class DocCodeGenState extends BaseCodeGenState {
3030
protected handleProgress(messenger: DocMessenger, action: SessionStateAction, detail?: string): void {
@@ -82,21 +82,37 @@ export class DocCodeGenState extends BaseCodeGenState {
8282
return new PromptRefusalException(codegenResult.codeGenerationRemainingIterationCount || 0)
8383
}
8484
case codegenResult.codeGenerationStatusDetail?.includes('Guardrails'): {
85-
return new DocServiceError(i18n('AWS.amazonq.doc.error.docGen.default'), 'GuardrailsException')
85+
return new ApiClientError(
86+
i18n('AWS.amazonq.doc.error.docGen.default'),
87+
'GetTaskAssistCodeGeneration',
88+
'GuardrailsException',
89+
400
90+
)
8691
}
8792
case codegenResult.codeGenerationStatusDetail?.includes('EmptyPatch'): {
8893
if (codegenResult.codeGenerationStatusDetail?.includes('NO_CHANGE_REQUIRED')) {
8994
return new NoChangeRequiredException()
9095
}
91-
return new DocServiceError(i18n('AWS.amazonq.doc.error.docGen.default'), 'EmptyPatchException')
96+
97+
return new LlmError(i18n('AWS.amazonq.doc.error.docGen.default'), {
98+
code: 'EmptyPatchException',
99+
})
92100
}
93101
case codegenResult.codeGenerationStatusDetail?.includes('Throttling'): {
94-
return new DocServiceError(i18n('AWS.amazonq.featureDev.error.throttling'), 'ThrottlingException')
102+
return new ApiClientError(
103+
i18n('AWS.amazonq.featureDev.error.throttling'),
104+
'GetTaskAssistCodeGeneration',
105+
'ThrottlingException',
106+
429
107+
)
95108
}
96109
default: {
97-
return new ToolkitError(i18n('AWS.amazonq.doc.error.docGen.default'), {
98-
code: 'DocGenerationFailed',
99-
})
110+
return new ApiServiceError(
111+
i18n('AWS.amazonq.doc.error.docGen.default'),
112+
'GetTaskAssistCodeGeneration',
113+
'UnknownException',
114+
500
115+
)
100116
}
101117
}
102118
}

‎packages/core/src/amazonqDoc/types.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,18 @@ export interface SessionStateAction extends FeatureDevSessionStateAction {
5555
folderPath?: string
5656
}
5757

58+
export enum MetricDataOperationName {
59+
StartDocGeneration = 'StartDocGeneration',
60+
EndDocGeneration = 'EndDocGeneration',
61+
}
62+
63+
export enum MetricDataResult {
64+
Success = 'Success',
65+
Fault = 'Fault',
66+
Error = 'Error',
67+
LlmFailure = 'LLMFailure',
68+
}
69+
5870
export {
5971
LLMResponseType,
6072
SessionStorage,

0 commit comments

Comments
 (0)