Merge branch 'main' into bryant/update-voice-fallback

Author: humford

fern/apis/api/openapi.json

@@ -6434,6 +6434,7 @@
             "type": "string",
             "description": "This is the OpenAI model that will be used.",
             "enum": [
+              "chatgpt-4o-latest",
               "o3-mini",
               "o1-preview",
               "o1-preview-2024-09-12",
@@ -6466,6 +6467,7 @@
             "type": "array",
             "description": "These are the fallback models that will be used if the primary model fails. This shouldn't be specified unless you have a specific reason to do so. Vapi will automatically find the fastest fallbacks that make sense.",
             "enum": [
+              "chatgpt-4o-latest",
               "o3-mini",
               "o1-preview",
               "o1-preview-2024-09-12",
@@ -6500,6 +6502,7 @@
             "items": {
               "type": "string",
               "enum": [
+                "chatgpt-4o-latest",
                 "o3-mini",
                 "o1-preview",
                 "o1-preview-2024-09-12",
@@ -7618,6 +7621,50 @@
           "voiceId"
         ]
       },
+      "CartesiaExperimentalControls": {
+        "type": "object",
+        "properties": {
+          "speed": {
+            "type": "string",
+            "enum": [
+              "slowest",
+              "slow",
+              "normal",
+              "fast",
+              "fastest"
+            ],
+            "example": "normal"
+          },
+          "emotion": {
+            "type": "string",
+            "enum": [
+              "anger:lowest",
+              "anger:low",
+              "anger:high",
+              "anger:highest",
+              "positivity:lowest",
+              "positivity:low",
+              "positivity:high",
+              "positivity:highest",
+              "surprise:lowest",
+              "surprise:low",
+              "surprise:high",
+              "surprise:highest",
+              "sadness:lowest",
+              "sadness:low",
+              "sadness:high",
+              "sadness:highest",
+              "curiosity:lowest",
+              "curiosity:low",
+              "curiosity:high",
+              "curiosity:highest"
+            ],
+            "example": [
+              "happiness:high"
+            ]
+          }
+        }
+      },
       "CartesiaVoice": {
         "type": "object",
         "properties": {
@@ -7628,6 +7675,10 @@
               "cartesia"
             ]
           },
+          "voiceId": {
+            "type": "string",
+            "description": "The ID of the particular voice you want to use."
+          },
           "model": {
             "type": "string",
             "description": "This is the model that will be used. This is optional and will default to the correct model for the voiceId.",
@@ -7661,6 +7712,14 @@
             ],
             "example": "en"
           },
+          "experimentalControls": {
+            "description": "Experimental controls for Cartesia voice generation",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/CartesiaExperimentalControls"
+              }
+            ]
+          },
           "chunkPlan": {
             "description": "This is the plan for chunking the model output before it is sent to the voice provider.",
             "allOf": [
@@ -7669,10 +7728,6 @@
               }
             ]
           },
-          "voiceId": {
-            "type": "string",
-            "description": "This is the provider-specific ID that will be used."
-          },
           "fallbackPlan": {
             "description": "This is the plan for voice provider fallbacks in the event that the primary voice provider fails.",
             "allOf": [
@@ -8657,6 +8712,10 @@
               "cartesia"
             ]
           },
+          "voiceId": {
+            "type": "string",
+            "description": "The ID of the particular voice you want to use."
+          },
           "model": {
             "type": "string",
             "description": "This is the model that will be used. This is optional and will default to the correct model for the voiceId.",
@@ -8690,9 +8749,13 @@
             ],
             "example": "en"
           },
-          "voiceId": {
-            "type": "string",
-            "description": "This is the provider-specific ID that will be used."
+          "experimentalControls": {
+            "description": "Experimental controls for Cartesia voice generation",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/CartesiaExperimentalControls"
+              }
+            ]
           },
           "chunkPlan": {
             "description": "This is the plan for chunking the model output before it is sent to the voice provider.",

fern/test/voice-testing.mdx

@@ -6,38 +6,36 @@ slug: /test/voice-testing
 
 ## Overview
 
-Voice Testing is an end-to-end feature that automates testing of your AI voice agents. Our platform simulates a call from an AI tester that interacts with your voice agent by following a pre-defined call script. After the call, the transcript is sent to a language model (LLM) along with your success criteria. The LLM then determines if the call met the defined objectives.
+Voice Testing is an end-to-end feature that automates testing of your AI voice agents. Our platform simulates a call from an AI tester that interacts with your voice agent by following a pre-defined call script. After the call, the transcript is sent to a language model (LLM) along with your evaluation rubric. The LLM then determines if the call met the defined objectives.
 
 ## Creating a Test Suite
 
 Begin by creating a Test Suite that organizes and executes multiple test cases.
 
 <Steps>
   ### Step 1: Create a New Test Suite
-    - Navigate to the **Test** tab in your dashboard and select **Voice Testing**.
+    - Navigate to the **Test** tab in your dashboard and select **Test Suites**.
     - Click the **Create Test Suite** button.
 
   ### Step 2: Define Test Suite Details
     - Enter a title for your Test Suite.
     - Select a phone number from your organization using the dropdown.
+    - Make sure the phone number has an assistant assigned to it (if not, navigate to Phone Numbers tab to complete that action).
 
   ### Step 3: Add Test Cases
     - Once your Test Suite is created, you will see a table where you can add test cases.
-    - Click **Add Test Case** to add a new test case (up to 50 can be added).
+    - Click **Add Test** to add a new test case (up to 50 can be added).
 
   ### Step 4: Configure Each Test Case
-    - **Caller Behavior:** Define how the testing agent should behave, including a detailed multi-step prompt that outlines the customer's intent, emotions, and interaction style.
-    - **Success Criteria:** List one or more questions that an LLM will use to evaluate if the call was successful.
+    - **Script:** Define how the testing agent should behave, including a detailed multi-step prompt to simulate how the customer should behave on the call.
+    - **Rubric** List one or more questions that an LLM will use to evaluate if the call was successful.
     - **Attempts:** Choose the number of times (up to 5) the test case should be executed each time the Test Suite is run.
 
   ### Step 5: Run and Review Tests
     - Click **Run Tests** to execute all test cases one by one.
     - While tests are running, you will see a loading state.
     - Upon completion, a table displays the outcomes with check marks (success) or x-marks (failure).
-    - Click on a test row to view detailed results: a dropdown shows each attempt, the LLM's reasoning, the transcript of the call, the defined caller behavior, and the success criteria.
-
-  ### Step 6: Export Results
-    - You can export the test results as a CSV file for further analysis.
+    - Click on a test row to view detailed results: a dropdown shows each attempt, the LLM's reasoning, the transcript of the call, the defined script, and the success rubric.
 </Steps>
 
 ## Test Execution and Evaluation
@@ -51,74 +49,56 @@ When you run a Test Suite, the following steps occur:
   - The number of attempts made.
   - The LLM's reasoning for each attempt.
   - The complete call transcript.
-  - The configured caller behavior and success criteria.
-- **CSV Export:** Results can be exported for additional review or compliance purposes.
+  - The configured script and rubric.
 
 ## Example Test Cases
 
-Below are three example test cases to illustrate how you can configure detailed caller behavior and success criteria.
-
-### Example 1: Account Inquiry
+Below are three example test cases to illustrate how you can configure detailed simulation scripts and evaluation rubrics.
 
-**Caller Behavior:**  
-Simulate a customer inquiring about their account status with growing concern as unexplained charges appear in their statement.  
+### Example 1: Billing Support
 
-**Example Prompt:**
-```md title="Example Prompt" wordWrap
-[Identity]
-You are a long-time bank customer with a keen eye for your financial details.
+In this example, we will simulate a customer who is frustrated and calling about a billing discrepancy.
 
-[Personality]
-Normally calm and polite, you become increasingly anxious when you notice discrepancies on your account statement. Your tone shifts from supportive to urgent as the conversation progresses.
-
-[Goals]
-Your primary objective is to clarify several unexplained charges by requesting a detailed breakdown of your recent transactions and ensuring your account balance is accurate.
-
-[Interaction Style]
-Begin the call by stating your name and expressing concern over unexpected charges. Ask straightforward questions and press for more details if the explanation is not satisfactory.
+**Script:** 
+```md title="Script" wordWrap
+1. Express anger over an unexpected charge and the current bill appearing unusually high.
+2. Try to get a detailed explanation, confirming whether an overcharge occurred, and understanding the steps for resolution.
+3. End the call.
 ```
 
-**Success Criteria:**
-```md title="Success Criteria" wordWrap
-1. The voice agent clearly presents the current account balance.
-2. The voice agent provides a detailed breakdown of recent transactions.
-3. The response addresses the customer's concerns in a calm and informative manner.
+**Rubric:**
+```md title="Rubric" wordWrap
+The voice agent acknowledges the billing discrepancy respectfully without dismissing the concern.
 ```
 
-### Example 2: Billing Support
-
-**Caller Behavior:**  
-Simulate a customer who is frustrated and calling about a billing discrepancy.  
+### Example 2: Account Inquiry
 
-**Example Prompt:**
-```txt title="Example Prompt" wordWrap
-[Identity]
-You are a loyal customer who has always trusted the billing process, but the current bill appears unusually high.
+Unlike in the previous example, this time we will provide a more free-form script for the test agent to follow.
 
-[Personality]
-Frustrated and assertive, you express anger over an unexpected charge while remaining focused on obtaining clarification.
+**Script:** 
+```md title="Script" wordWrap
+Simulate a customer inquiring about their account status with growing concern as unexplained charges appear in their statement.  
 
-[Goals]
-Your goal is to understand the discrepancy in your bill by obtaining a detailed explanation, confirming whether an overcharge occurred, and understanding the steps for resolution.
+Your primary objective is to clarify several unexplained charges by requesting a detailed breakdown of your recent transactions and ensuring your account balance is accurate.
 
-[Interaction Style]
-Start the call by clearly stating your billing concern, describing the specific overcharge, and requesting a comprehensive explanation with resolution options.
+Begin the call by stating your name and expressing concern over unexpected charges. Ask straightforward questions and press for more details if the explanation is not satisfactory.
 ```
 
-**Success Criteria:**
-```md title="Success Criteria" wordWrap
-1. The voice agent acknowledges the billing discrepancy respectfully without dismissing the concern.
-2. The agent provides a clear explanation of the charges, detailing possible reasons for the discrepancy.
-3. The conversation concludes with a proposed solution or a clear escalation plan to address the overcharge.
+**Rubric:**
+```md title="Rubric" wordWrap
+1. The voice agent clearly presents the current account balance.
+2. The voice agent provides a detailed breakdown of recent transactions.
+3. The response addresses the customer's concerns in a calm and informative manner.
 ```
 
 ### Example 3: Appointment Scheduling
 
-**Caller Behavior:**  
+This time, we will spin up an even more detailed personality for the test agent. By showing these varied styles of scripts, we hope to show the flexibility of the Voice Testing feature and how you can use it to meet your testing needs.
+
+**Script:** 
+```md title="Script" wordWrap
 Simulate a customer trying to schedule an appointment with a hint of urgency due to previous delays.  
 
-**Example Prompt:**
-```md title="Example Prompt" wordWrap
 [Identity]
 You are an organized customer who values efficiency and punctuality.
 
@@ -132,8 +112,8 @@ Your goal is to secure an appointment at your preferred time, while remaining fl
 Begin the call by stating your need for an appointment, specifying a preferred date and time (e.g., next Monday at 3 PM). Request clear confirmation of your slot, and if unavailable, ask for suitable alternatives.
 ```
 
-**Success Criteria:**
-```md title="Success Criteria" wordWrap
+**Rubric:**
+```md title="Rubric" wordWrap
 1. The voice agent confirms the requested appointment time clearly and accurately.
 2. The agent reiterates the appointment details to ensure clarity.
 3. The scheduling process ends with a definitive confirmation message of the booked appointment.