fix(amazonq): filter out frequent client error from codegeneration telemetry (#6521)

## Problem
We are mistakenly counting several client-side failures as service
faults, which is affecting our availability metrics.

## Solution
Exclude frequent client errors from faults, they should be errors.

there are multiple layers of errors, some of those are categorized to
toolkit level errors, for example this part:
```
case MonthlyConversationLimitError.name:
case CodeIterationLimitError.name:
case PromptRefusalException.name:
case NoChangeRequiredException.name:
```
Others are defaulting to ToolkitError with messages.

Ideally we should add more error models to toolkit, but for now to make
a quick fix, I just looked into the error msg, similar to what we do
here:
https://github.com/aws/aws-toolkit-vscode/blob/02d21a2fafc7479fb398ea078aa3e3adf7c62e7a/packages/core/src/amazonqFeatureDev/client/featureDev.ts#L198



---

- Treat all work as PUBLIC. Private `feature/x` branches will not be
squash-merged at release time.
- Your code changes must meet the guidelines in
[CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines).
- License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

---------

Co-authored-by: Yuxin Lin <[email protected]>

Author: linyuxi-0511

packages/core/src/amazonqFeatureDev/client/featureDev.ts

@@ -10,7 +10,7 @@ import { ServiceOptions } from '../../shared/awsClientBuilder'
 import globals from '../../shared/extensionGlobals'
 import { getLogger } from '../../shared/logger/logger'
 import * as FeatureDevProxyClient from './featuredevproxyclient'
-import { featureName } from '../constants'
+import { featureName, startTaskAssistLimitReachedMessage } from '../constants'
 import { CodeReference } from '../../amazonq/webview/ui/connector'
 import {
     ApiError,
@@ -185,10 +185,7 @@ export class FeatureDevClient {
             )
             if (isAwsError(e)) {
                 // API Front-end will throw Throttling if conversation limit is reached. API Front-end monitors StartCodeGeneration for throttling
-                if (
-                    e.code === 'ThrottlingException' &&
-                    e.message.includes('StartTaskAssistCodeGeneration reached for this month.')
-                ) {
+                if (e.code === 'ThrottlingException' && e.message.includes(startTaskAssistLimitReachedMessage)) {
                     throw new MonthlyConversationLimitError(e.message)
                 }
                 // BE service will throw ServiceQuota if code generation iteration limit is reached

packages/core/src/amazonqFeatureDev/constants.ts

@@ -20,6 +20,12 @@ export const generateDevFilePrompt =
 // Max allowed size for file collection
 export const maxRepoSizeBytes = 200 * 1024 * 1024
 
+export const startCodeGenClientErrorMessages = ['Improperly formed request', 'Resource not found']
+export const startTaskAssistLimitReachedMessage = 'StartTaskAssistCodeGeneration reached for this month.'
+export const clientErrorMessages = [
+    'The folder you chose did not contain any source files in a supported language. Choose another folder and try again.',
+]
+
 // License text that's used in codewhisperer reference log
 export const referenceLogText = (reference: CodeReference) =>
     `[${new Date().toLocaleString()}] Accepted recommendation from Amazon Q. Code provided with reference under <a href="${LicenseUtil.getLicenseHtml(

packages/core/src/amazonqFeatureDev/controllers/chat/controller.ts

@@ -15,6 +15,7 @@ import {
     createUserFacingErrorMessage,
     denyListedErrors,
     FeatureDevServiceError,
+    isAPIClientError,
     MonthlyConversationLimitError,
     NoChangeRequiredException,
     PrepareRepoFailedError,
@@ -555,12 +556,18 @@ export class FeatureDevController {
                         result = MetricDataResult.Fault
                     }
                     break
+                case MonthlyConversationLimitError.name:
+                case CodeIterationLimitError.name:
                 case PromptRefusalException.name:
                 case NoChangeRequiredException.name:
                     result = MetricDataResult.Error
                     break
                 default:
-                    result = MetricDataResult.Fault
+                    if (isAPIClientError(err)) {
+                        result = MetricDataResult.Error
+                    } else {
+                        result = MetricDataResult.Fault
+                    }
                     break
             }
 

packages/core/src/amazonqFeatureDev/errors.ts

@@ -4,7 +4,12 @@
  */
 
 import { ToolkitError } from '../shared/errors'
-import { featureName } from './constants'
+import {
+    featureName,
+    clientErrorMessages,
+    startCodeGenClientErrorMessages,
+    startTaskAssistLimitReachedMessage,
+} from './constants'
 import { uploadCodeError } from './userFacingText'
 import { i18n } from '../shared/i18n-helper'
 
@@ -140,3 +145,12 @@ export function createUserFacingErrorMessage(message: string) {
     }
     return message
 }
+
+export function isAPIClientError(error: { code?: string; message: string }): boolean {
+    return (
+        (error.code === 'StartCodeGenerationFailed' &&
+            startCodeGenClientErrorMessages.some((msg: string) => error.message.includes(msg))) ||
+        clientErrorMessages.some((msg: string) => error.message.includes(msg)) ||
+        error.message.includes(startTaskAssistLimitReachedMessage)
+    )
+}

packages/core/src/test/amazonqFeatureDev/controllers/chat/controller.test.ts

@@ -24,6 +24,7 @@ import {
     ContentLengthError,
     createUserFacingErrorMessage,
     FeatureDevServiceError,
+    isAPIClientError,
     MonthlyConversationLimitError,
     NoChangeRequiredException,
     PrepareRepoFailedError,
@@ -471,12 +472,17 @@ describe('Controller', () => {
                 ['EmptyPatchException', MetricDataResult.LlmFailure],
                 [PromptRefusalException.name, MetricDataResult.Error],
                 [NoChangeRequiredException.name, MetricDataResult.Error],
+                [MonthlyConversationLimitError.name, MetricDataResult.Error],
+                [CodeIterationLimitError.name, MetricDataResult.Error],
             ])
 
             function getMetricResult(error: ToolkitError): MetricDataResult {
                 if (error instanceof FeatureDevServiceError && error.code) {
                     return errorResultMapping.get(error.code) ?? MetricDataResult.Error
                 }
+                if (isAPIClientError(error)) {
+                    return MetricDataResult.Error
+                }
                 return errorResultMapping.get(error.constructor.name) ?? MetricDataResult.Fault
             }