updated answer confidence definition

src/lib/intelligent-support/schemas.ts

@@ -36,18 +36,52 @@ export const ProvideLinksToolSchema = z.object({
   links: LinksSchema,
 });
 
-const KnownAnswerConfidence = z.enum(['very_confident', 'somewhat_confident', 'not_confident', 'no_sources', 'other']);
+const KnownAnswerConfidence = z
+.union([
+  z.literal("very_confident").describe(`
+  The AI Assistant provided a complete and direct answer to all parts of the User Question.
+  The answer fully resolved the issue without requiring any further action from the User.
+  Every part of the answer was cited from the information sources.
+  The assistant did not ask for more information or provide options requiring User action.
+  This is the highest Answer Confidence level and should be used sparingly.
+`),
+  z.literal("somewhat_confident").describe(`
+  The AI Assistant provided a complete and direct answer to the User Question, but the answer contained minor caveats or uncertainties. 
 
-const AnswerConfidence = z.union([KnownAnswerConfidence, z.string()]);
+  Examples:
+  • The AI Assistant asked follow-up questions to the User
+  • The AI Assistant requested additional information from the User
+  • The AI Assistant suggested uncertainty in the answer
+  • The AI Assistant answered the question but mentioned potential exceptions
+`),
+  z.literal("not_confident").describe(`
+  The AI Assistant tried to answer the User Question but did not fully resolve it.
+  The assistant provided options requiring further action from the User, asked for more information, showed uncertainty,
+  suggested the user contact support or provided contact information, or provided an indirect or incomplete answer.
+  This is the most common Answer Confidence level.
 
-const AIAnnotationsToolSchema = z
-  .object({
-    answerConfidence: AnswerConfidence,
-  })
-  .passthrough();
+  Examples:
+  • The AI Assistant provided a general answer not directly related to the User Question
+  • The AI Assistant said to reach out to support or provided an email address or contact information
+  • The AI Assistant provided options that require further action from the User to resolve the issue
+`),
+  z.literal("no_sources").describe(`
+  The AI Assistant did not use or cite any sources from the information sources to answer the User Question.
+`),
+  z.literal("other").describe(`
+  The User Question is unclear or unrelated to the subject matter.
+`),
+])
+.describe(
+  "A measure of how confidently the AI Assistant completely and directly answered the User Question."
+);
+
+const AnswerConfidence = z.union([KnownAnswerConfidence, z.string()]);
+const AnswerConfidenceExplanation = z.string().describe("A brief few word justification of why a specific confidence level was chosen.");
 
-export const ProvideAIAnnotationsToolSchema = z.object({
-  aiAnnotations: AIAnnotationsToolSchema,
+export const ProvideAnswerConfidenceToolSchema = z.object({
+  explanation: AnswerConfidenceExplanation,
+  answerConfidence: AnswerConfidence,
 });
 
 export const ProvideRecordsConsideredToolSchema = z.object({

src/lib/intelligent-support/support.ts

@@ -1,5 +1,5 @@
 import 'server-only';
-import { ProvideAIAnnotationsToolSchema, ProvideLinksToolSchema } from './schemas';
+import { ProvideAnswerConfidenceToolSchema, ProvideLinksToolSchema } from './schemas';
 
 import type { CoreMessage } from 'ai';
 import type { z } from 'zod';
@@ -40,8 +40,8 @@ export const generateQaModeResponse = async ({
       provideRecordsConsidered: {
         parameters: ProvideRecordsConsideredToolSchema,
       },
-      provideAIAnnotations: {
-        parameters: ProvideAIAnnotationsToolSchema,
+      provideAnswerConfidence: {
+        parameters: ProvideAnswerConfidenceToolSchema,
       },
       provideLinks: {
         parameters: ProvideLinksToolSchema,
@@ -50,16 +50,16 @@ export const generateQaModeResponse = async ({
     toolChoice: 'auto',
   });
 
-  const aiAnnotations = toolCalls.find(toolCall => toolCall.toolName === 'provideAIAnnotations')?.args
-    .aiAnnotations as z.infer<typeof ProvideAIAnnotationsToolSchema>['aiAnnotations'];
+  const answerConfidence = toolCalls.find(toolCall => toolCall.toolName === 'provideAnswerConfidence')?.args
+    .answerConfidence as z.infer<typeof ProvideAnswerConfidenceToolSchema>['answerConfidence'];
   const recordsConsidered = toolCalls.find(toolCall => toolCall.toolName === 'provideRecordsConsidered')?.args
     .recordsConsidered as z.infer<typeof ProvideRecordsConsideredToolSchema>['recordsConsidered'];
   const links = toolCalls.find(toolCall => toolCall.toolName === 'provideLinks')?.args.links as z.infer<
     typeof ProvideLinksToolSchema
   >['links'];
 
   return {
-    aiAnnotations,
+    answerConfidence,
     text,
     recordsConsidered,
     links,