Improves AI error messages more (#4227)

Author: eamodio

docs/telemetry-events.md

@@ -120,6 +120,7 @@
   'duration': number,
   'failed.cancelled.reason': 'large-prompt',
   'failed.error': string,
+  'failed.error.detail': string,
   'failed.reason': 'user-declined' | 'user-cancelled' | 'error',
   'input.length': number,
   'model.id': string,
@@ -150,6 +151,7 @@
   'duration': number,
   'failed.cancelled.reason': 'large-prompt',
   'failed.error': string,
+  'failed.error.detail': string,
   'failed.reason': 'user-declined' | 'user-cancelled' | 'error',
   'input.length': number,
   'model.id': string,
@@ -179,6 +181,7 @@ or
   'duration': number,
   'failed.cancelled.reason': 'large-prompt',
   'failed.error': string,
+  'failed.error.detail': string,
   'failed.reason': 'user-declined' | 'user-cancelled' | 'error',
   'input.length': number,
   'model.id': string,
@@ -207,6 +210,7 @@ or
   'duration': number,
   'failed.cancelled.reason': 'large-prompt',
   'failed.error': string,
+  'failed.error.detail': string,
   'failed.reason': 'user-declined' | 'user-cancelled' | 'error',
   'input.length': number,
   'model.id': string,
@@ -235,6 +239,7 @@ or
   'duration': number,
   'failed.cancelled.reason': 'large-prompt',
   'failed.error': string,
+  'failed.error.detail': string,
   'failed.reason': 'user-declined' | 'user-cancelled' | 'error',
   'input.length': number,
   'model.id': string,
@@ -263,6 +268,7 @@ or
   'duration': number,
   'failed.cancelled.reason': 'large-prompt',
   'failed.error': string,
+  'failed.error.detail': string,
   'failed.reason': 'user-declined' | 'user-cancelled' | 'error',
   'input.length': number,
   'model.id': string,

src/constants.telemetry.ts

@@ -328,6 +328,7 @@ interface AIEventDataBase {
 	'failed.reason'?: 'user-declined' | 'user-cancelled' | 'error';
 	'failed.cancelled.reason'?: 'large-prompt';
 	'failed.error'?: string;
+	'failed.error.detail'?: string;
 }
 
 interface AIExplainEvent extends AIEventDataBase {

src/errors.ts

@@ -278,16 +278,17 @@ export class RequiresIntegrationError extends Error {
 }
 
 export const enum AIErrorReason {
+	DeniedByOrganization,
+	DeniedByUser,
+	NoEntitlement,
 	NoRequestData,
-	Entitlement,
-	RequestTooLarge,
-	UserQuotaExceeded,
 	RateLimitExceeded,
 	RateLimitOrFundsExceeded,
-	ServiceCapacityExceeded,
+	RequestTooLarge,
 	ModelNotSupported,
-	ModelUserUnauthorized,
-	ModelUserDeniedAccess,
+	ServiceCapacityExceeded,
+	Unauthorized,
+	UserQuotaExceeded,
 }
 
 export class AIError extends Error {
@@ -297,7 +298,7 @@ export class AIError extends Error {
 	constructor(reason: AIErrorReason, original?: Error) {
 		let message;
 		switch (reason) {
-			case AIErrorReason.Entitlement:
+			case AIErrorReason.NoEntitlement:
 				message = 'You do not have the required entitlement to use this feature';
 				break;
 			case AIErrorReason.RequestTooLarge:
@@ -321,11 +322,14 @@ export class AIError extends Error {
 			case AIErrorReason.ModelNotSupported:
 				message = 'Model not supported for this request';
 				break;
-			case AIErrorReason.ModelUserUnauthorized:
-				message = 'User is not authorized to use the specified model';
+			case AIErrorReason.Unauthorized:
+				message = 'You are not authorized to use the specified provider or model';
+				break;
+			case AIErrorReason.DeniedByOrganization:
+				message = 'Your organization has denied access to the specified provider or model';
 				break;
-			case AIErrorReason.ModelUserDeniedAccess:
-				message = 'User denied access to the specified model';
+			case AIErrorReason.DeniedByUser:
+				message = 'You have denied access to the specified provider or model';
 				break;
 			default:
 				message = original?.message ?? 'An unknown error occurred';

src/plus/ai/aiProviderService.ts

@@ -916,26 +916,34 @@ export class AIProviderService implements Disposable {
 
 			return result;
 		} catch (ex) {
-			this.container.telemetry.sendEvent(
-				telementry.key,
-				{
-					...telementry.data,
-					duration: Date.now() - start,
-					...(ex instanceof CancellationError
-						? { 'failed.reason': 'user-cancelled' }
-						: { 'failed.reason': 'error', 'failed.error': String(ex) }),
-				},
-				source,
-			);
+			if (ex instanceof CancellationError) {
+				this.container.telemetry.sendEvent(
+					telementry.key,
+					{ ...telementry.data, duration: Date.now() - start, 'failed.reason': 'user-cancelled' },
+					source,
+				);
 
-			if (ex instanceof CancellationError) return undefined;
+				return undefined;
+			}
 			if (ex instanceof AIError) {
+				this.container.telemetry.sendEvent(
+					telementry.key,
+					{
+						...telementry.data,
+						duration: Date.now() - start,
+						// eslint-disable-next-line @typescript-eslint/no-base-to-string
+						'failed.error': String(ex),
+						'failed.error.detail': String(ex.original),
+					},
+					source,
+				);
+
 				switch (ex.reason) {
 					case AIErrorReason.NoRequestData:
 						void window.showErrorMessage(ex.message);
 						return undefined;
 
-					case AIErrorReason.Entitlement: {
+					case AIErrorReason.NoEntitlement: {
 						const sub = await this.container.subscription.getSubscription();
 
 						const plan = isSubscriptionPaid(sub)
@@ -1021,7 +1029,7 @@ export class AIProviderService implements Disposable {
 						}
 						return undefined;
 					}
-					case AIErrorReason.ModelUserUnauthorized: {
+					case AIErrorReason.Unauthorized: {
 						const switchModel: MessageItem = { title: 'Switch Model' };
 						const result = await window.showErrorMessage(
 							'You do not have access to the selected model. Please select a different model and try again.',
@@ -1032,7 +1040,7 @@ export class AIProviderService implements Disposable {
 						}
 						return undefined;
 					}
-					case AIErrorReason.ModelUserDeniedAccess: {
+					case AIErrorReason.DeniedByUser: {
 						const switchModel: MessageItem = { title: 'Switch Model' };
 						const result = await window.showErrorMessage(
 							'You have denied access to the selected model. Please provide access or select a different model, and then try again.',
@@ -1048,6 +1056,16 @@ export class AIProviderService implements Disposable {
 				return undefined;
 			}
 
+			this.container.telemetry.sendEvent(
+				telementry.key,
+				{
+					...telementry.data,
+					duration: Date.now() - start,
+					'failed.error': String(ex),
+					'failed.error.detail': ex.original ? String(ex.original) : undefined,
+				},
+				source,
+			);
 			throw ex;
 		}
 	}

src/plus/ai/gitkrakenProvider.ts

@@ -135,6 +135,7 @@ export class GitKrakenProvider extends OpenAICompatibleProvider<typeof provider.
 			case 403:
 				// CodeAuthorization      = "403.1"
 				// CodeEntitlement        = "403.2"
+				// CodeOrganization       = "403.3"
 
 				// Entitlement Error
 				if (code === 2) {
@@ -152,10 +153,19 @@ export class GitKrakenProvider extends OpenAICompatibleProvider<typeof provider.
 
 					throw new AIError(
 						// If there is an `entitlementValue` then we are over the limit, otherwise it is an entitlement error
-						data?.entitlementValue ? AIErrorReason.UserQuotaExceeded : AIErrorReason.Entitlement,
+						data?.entitlementValue ? AIErrorReason.UserQuotaExceeded : AIErrorReason.NoEntitlement,
+						new Error(`(${this.name}) ${status}.${code}: ${message}`),
+					);
+				} else if (code === 3) {
+					throw new AIError(
+						AIErrorReason.DeniedByOrganization,
 						new Error(`(${this.name}) ${status}.${code}: ${message}`),
 					);
 				}
+				throw new AIError(
+					AIErrorReason.Unauthorized,
+					new Error(`(${this.name}) ${status}.${code}: ${message}`),
+				);
 				throw new Error(`(${this.name}) ${status}.${code}: ${message}`);
 			case 404:
 				// CodeNotFound           = "404.1"

src/plus/ai/openAICompatibleProvider.ts

@@ -165,7 +165,7 @@ export abstract class OpenAICompatibleProvider<T extends AIProviders> implements
 	): Promise<{ retry: true; maxInputTokens: number }> {
 		if (rsp.status === 404) {
 			throw new AIError(
-				AIErrorReason.ModelUserUnauthorized,
+				AIErrorReason.Unauthorized,
 				new Error(`Your API key doesn't seem to have access to the selected '${model.id}' model`),
 			);
 		}

src/plus/ai/vscodeProvider.ts

@@ -123,7 +123,7 @@ export class VSCodeAIProvider implements AIProvider<typeof provider.id> {
 
 				if (ex instanceof Error && 'code' in ex && ex.code === 'NoPermissions') {
 					Logger.error(ex, scope, `User denied access to ${model.provider.name}`);
-					throw new AIError(AIErrorReason.ModelUserDeniedAccess, ex);
+					throw new AIError(AIErrorReason.DeniedByUser, ex);
 				}
 
 				if (ex instanceof Error && 'cause' in ex && ex.cause instanceof Error) {