telemetry(amazonq): /dev error classification aws#6772

## Problem

Today, telemetry emitted from the feature development is classified as
error, fault, or LLM-based failure across various parts of the code
base. This makes it difficult to know all the error types that exist and
how to classify them appropriately when emitting metrics.

## Solution
The solution here allows different exceptions to raise from different
parts of the code base and uses inheritance to model each error as a
client error, service error, or LLM-based error. When emitting
telemetry, we now only need to check whether the error received extends
from one of these 3 to know how to classify the telemetry.

Author: neilk-aws

packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts

@@ -11,11 +11,11 @@ import {
     maxRepoSizeBytes,
 } from 'aws-core-vscode/amazonqFeatureDev'
 import { assertTelemetry, getWorkspaceFolder, TestFolder } from 'aws-core-vscode/test'
-import { fs, AmazonqCreateUpload, ZipStream } from 'aws-core-vscode/shared'
+import { fs, AmazonqCreateUpload, ZipStream, ContentLengthError } from 'aws-core-vscode/shared'
 import { MetricName, Span } from 'aws-core-vscode/telemetry'
 import sinon from 'sinon'
 import { CodeWhispererSettings } from 'aws-core-vscode/codewhisperer'
-import { ContentLengthError, CurrentWsFolders } from 'aws-core-vscode/amazonq'
+import { CurrentWsFolders } from 'aws-core-vscode/amazonq'
 import path from 'path'
 
 const testDevfilePrepareRepo = async (devfileEnabled: boolean) => {

packages/core/src/amazonq/errors.ts

@@ -9,10 +9,13 @@
  * When thrown from common components, individual agents can catch and transform this error
  * to provide their own customized error messages.
  */
-import { ToolkitError } from '../shared/errors'
+import { ErrorInformation, ToolkitError } from '../shared/errors'
 
-export class ContentLengthError extends ToolkitError {
-    constructor(message: string) {
-        super(message, { code: 'ContentLengthError' })
+/**
+ * Errors extending this class are considered "LLM failures" in service metrics.
+ */
+export class LlmError extends ToolkitError {
+    constructor(message: string, info: ErrorInformation = {}) {
+        super(message, info)
     }
 }

packages/core/src/amazonq/index.ts

@@ -44,7 +44,6 @@ export { ExtensionMessage } from '../amazonq/webview/ui/commands'
 export { CodeReference } from '../codewhispererChat/view/connector/connector'
 export { extractAuthFollowUp } from './util/authUtils'
 export { Messenger } from './commons/connector/baseMessenger'
-export { ContentLengthError } from './errors'
 import { FeatureContext } from '../shared/featureConfig'
 
 /**

packages/core/src/amazonq/util/files.ts

@@ -16,7 +16,7 @@ import { PrepareRepoFailedError } from '../../amazonqFeatureDev/errors'
 import { getLogger } from '../../shared/logger/logger'
 import { maxFileSizeBytes } from '../../amazonqFeatureDev/limits'
 import { CurrentWsFolders, DeletedFileInfo, NewFileInfo, NewFileZipContents } from '../../amazonqDoc/types'
-import { hasCode, ToolkitError } from '../../shared/errors'
+import { ContentLengthError, hasCode, ToolkitError } from '../../shared/errors'
 import { AmazonqCreateUpload, Span, telemetry as amznTelemetry, telemetry } from '../../shared/telemetry/telemetry'
 import { maxRepoSizeBytes } from '../../amazonqFeatureDev/constants'
 import { isCodeFile } from '../../shared/filetypes'
@@ -28,7 +28,6 @@ import { ZipStream } from '../../shared/utilities/zipStream'
 import { isPresent } from '../../shared/utilities/collectionUtils'
 import { AuthUtil } from '../../codewhisperer/util/authUtil'
 import { TelemetryHelper } from '../util/telemetryHelper'
-import { ContentLengthError } from '../errors'
 
 export const SvgFileExtension = '.svg'
 

packages/core/src/amazonqDoc/session/session.ts

@@ -30,8 +30,8 @@ import fs from '../../shared/fs/fs'
 import globals from '../../shared/extensionGlobals'
 import { extensionVersion } from '../../shared/vscode/env'
 import { getLogger } from '../../shared/logger/logger'
+import { ContentLengthError as CommonContentLengthError } from '../../shared/errors'
 import { ContentLengthError } from '../errors'
-import { ContentLengthError as CommonAmazonQContentLengthError } from '../../amazonq/errors'
 
 export class Session {
     private _state?: SessionState | Omit<SessionState, 'uploadId'>
@@ -152,7 +152,7 @@ export class Session {
 
             return resp.interaction
         } catch (e) {
-            if (e instanceof CommonAmazonQContentLengthError) {
+            if (e instanceof CommonContentLengthError) {
                 getLogger().debug(`Content length validation failed: ${e.message}`)
                 throw new ContentLengthError()
             }

packages/core/src/amazonqFeatureDev/client/featureDev.ts

@@ -14,12 +14,14 @@ import { featureName, startTaskAssistLimitReachedMessage } from '../constants'
 import { CodeReference } from '../../amazonq/webview/ui/connector'
 import {
     ApiError,
+    ApiServiceError,
     CodeIterationLimitError,
     ContentLengthError,
+    FeatureDevServiceError,
     MonthlyConversationLimitError,
     UnknownApiError,
 } from '../errors'
-import { ToolkitError, isAwsError } from '../../shared/errors'
+import { isAwsError } from '../../shared/errors'
 import { getCodewhispererConfig } from '../../codewhisperer/client/codewhisperer'
 import { createCodeWhispererChatStreamingClient } from '../../shared/clients/codewhispererChatClient'
 import { getClientId, getOptOutPreference, getOperatingSystem } from '../../shared/telemetry/util'
@@ -93,7 +95,7 @@ export class FeatureDevClient implements FeatureClient {
                 ) {
                     throw new MonthlyConversationLimitError(e.message)
                 }
-                throw new ApiError(e.message, 'CreateConversation', e.code, e.statusCode ?? 400)
+                throw ApiError.of(e.message, 'CreateConversation', e.code, e.statusCode ?? 500)
             }
 
             throw new UnknownApiError(e instanceof Error ? e.message : 'Unknown error', 'CreateConversation')
@@ -136,7 +138,7 @@ export class FeatureDevClient implements FeatureClient {
                 if (e.code === 'ValidationException' && e.message.includes('Invalid contentLength')) {
                     throw new ContentLengthError()
                 }
-                throw new ApiError(e.message, 'CreateUploadUrl', e.code, e.statusCode ?? 400)
+                throw ApiError.of(e.message, 'CreateUploadUrl', e.code, e.statusCode ?? 500)
             }
 
             throw new UnknownApiError(e instanceof Error ? e.message : 'Unknown error', 'CreateUploadUrl')
@@ -198,8 +200,10 @@ export class FeatureDevClient implements FeatureClient {
                 ) {
                     throw new CodeIterationLimitError()
                 }
+                throw ApiError.of(e.message, 'StartTaskAssistCodeGeneration', e.code, e.statusCode ?? 500)
             }
-            throw new ToolkitError((e as Error).message, { code: 'StartCodeGenerationFailed' })
+
+            throw new UnknownApiError(e instanceof Error ? e.message : 'Unknown error', 'StartTaskAssistCodeGeneration')
         }
     }
 
@@ -220,7 +224,12 @@ export class FeatureDevClient implements FeatureClient {
                     (e as any).requestId
                 }`
             )
-            throw new ToolkitError((e as Error).message, { code: 'GetCodeGenerationFailed' })
+
+            if (isAwsError(e)) {
+                throw ApiError.of(e.message, 'GetTaskAssistCodeGeneration', e.code, e.statusCode ?? 500)
+            }
+
+            throw new UnknownApiError(e instanceof Error ? e.message : 'Unknown error', 'GetTaskAssistCodeGeneration')
         }
     }
 
@@ -235,7 +244,12 @@ export class FeatureDevClient implements FeatureClient {
             const archiveResponse = await streamingClient.exportResultArchive(params)
             const buffer: number[] = []
             if (archiveResponse.body === undefined) {
-                throw new ToolkitError('Empty response from CodeWhisperer Streaming service.')
+                throw new ApiServiceError(
+                    'Empty response from CodeWhisperer Streaming service.',
+                    'ExportResultArchive',
+                    'EmptyResponse',
+                    500
+                )
             }
             for await (const chunk of archiveResponse.body) {
                 if (chunk.internalServerException !== undefined) {
@@ -274,7 +288,12 @@ export class FeatureDevClient implements FeatureClient {
                     (e as any).requestId
                 }`
             )
-            throw new ToolkitError((e as Error).message, { code: 'ExportResultArchiveFailed' })
+
+            if (isAwsError(e)) {
+                throw ApiError.of(e.message, 'ExportResultArchive', e.code, e.statusCode ?? 500)
+            }
+
+            throw new FeatureDevServiceError(e instanceof Error ? e.message : 'Unknown error', 'ExportResultArchive')
         }
     }
 

packages/core/src/amazonqFeatureDev/controllers/chat/controller.ts

@@ -15,7 +15,7 @@ import {
     createUserFacingErrorMessage,
     denyListedErrors,
     FeatureDevServiceError,
-    isAPIClientError,
+    getMetricResult,
     MonthlyConversationLimitError,
     NoChangeRequiredException,
     PrepareRepoFailedError,
@@ -550,34 +550,7 @@ export class FeatureDevController {
             this.messenger.sendUpdatePlaceholder(tabID, i18n('AWS.amazonq.featureDev.pillText.selectOption'))
         } catch (err: any) {
             getLogger().error(`${featureName}: Error during code generation: ${err}`)
-
-            let result: string
-            switch (err.constructor.name) {
-                case FeatureDevServiceError.name:
-                    if (err.code === 'EmptyPatchException') {
-                        result = MetricDataResult.LlmFailure
-                    } else if (err.code === 'GuardrailsException' || err.code === 'ThrottlingException') {
-                        result = MetricDataResult.Error
-                    } else {
-                        result = MetricDataResult.Fault
-                    }
-                    break
-                case MonthlyConversationLimitError.name:
-                case CodeIterationLimitError.name:
-                case PromptRefusalException.name:
-                case NoChangeRequiredException.name:
-                    result = MetricDataResult.Error
-                    break
-                default:
-                    if (isAPIClientError(err)) {
-                        result = MetricDataResult.Error
-                    } else {
-                        result = MetricDataResult.Fault
-                    }
-                    break
-            }
-
-            await session.sendMetricDataTelemetry(MetricDataOperationName.EndCodeGeneration, result)
+            await session.sendMetricDataTelemetry(MetricDataOperationName.EndCodeGeneration, getMetricResult(err))
             throw err
         } finally {
             // Finish processing the event