Support different formats of data within context field of test cases

.github/workflows/test_core.yml

@@ -61,6 +61,9 @@ jobs:
       - name: Install dev dependencies
         run: poetry install --no-interaction --with dev
 
+      - name: Install other dependencies
+        run: poetry run pip install beautifulsoup4
+
       #----------------------------------------------
       #              run test suite
       #----------------------------------------------

a.py

@@ -1,84 +1,84 @@
-"""
-Example script demonstrating how to use DeepEval's PromptOptimizer.
-"""
-
-from openai import OpenAI
-from deepeval.optimizer import PromptOptimizer
-from deepeval.prompt import Prompt
-from deepeval.dataset import Golden
-from deepeval.metrics import AnswerRelevancyMetric
-
-# Initialize OpenAI client
-client = OpenAI()
-
-
-def model_callback(prompt: Prompt, golden: Golden) -> str:
-    """
-    Callback function that runs your LLM with the optimized prompt.
-    This is called during scoring to evaluate how well the prompt performs.
-    """
-    # Interpolate the prompt template with the golden's input
-    final_prompt = prompt.interpolate(query=golden.input)
-
-    # Call your LLM
-    response = client.chat.completions.create(
-        model="gpt-4o-mini",
-        messages=[{"role": "user", "content": final_prompt}],
-    )
-
-    return response.choices[0].message.content
-
-
-# Define your initial prompt template (intentionally bad for testing optimization)
-prompt = Prompt(
-    text_template="""idk maybe try to respond to this thing if u want lol
-
-{query}
-
-whatever:"""
-)
-
-# Define your evaluation dataset (goldens)
-goldens = [
-    Golden(
-        input="What is the capital of France?",
-        expected_output="Paris",
-    ),
-    Golden(
-        input="Who wrote Romeo and Juliet?",
-        expected_output="William Shakespeare",
-    ),
-    Golden(
-        input="What is the chemical symbol for gold?",
-        expected_output="Au",
-    ),
-    Golden(
-        input="In what year did World War II end?",
-        expected_output="1945",
-    ),
-]
-
-# Define metrics to optimize for
-metrics = [AnswerRelevancyMetric(threshold=0.7)]
-
-from deepeval.optimizer.configs import DisplayConfig
-from deepeval.optimizer.algorithms import GEPA
-
-# Create the optimizer
-optimizer = PromptOptimizer(
-    model_callback=model_callback,
-    metrics=metrics,
-    optimizer_model="gpt-4o",  # Model used for rewriting prompts
-    display_config=DisplayConfig(announce_ties=True),
-    algorithm=GEPA(iterations=1),
-)
-
-# Run optimization
-optimized_prompt = optimizer.optimize(prompt=prompt, goldens=goldens)
-
-# Print results
-print("\n" + "=" * 60)
-print("OPTIMIZATION COMPLETE")
-print("=" * 60)
-print(f"\nOriginal prompt:\n{prompt.text_template}")
-print(f"\nOptimized prompt:\n{optimized_prompt.text_template}")
+# """
+# Example script demonstrating how to use DeepEval's PromptOptimizer.
+# """
+
+# from openai import OpenAI
+# from deepeval.optimizer import PromptOptimizer
+# from deepeval.prompt import Prompt
+# from deepeval.dataset import Golden
+# from deepeval.metrics import AnswerRelevancyMetric
+
+# # Initialize OpenAI client
+# client = OpenAI()
+
+
+# def model_callback(prompt: Prompt, golden: Golden) -> str:
+#     """
+#     Callback function that runs your LLM with the optimized prompt.
+#     This is called during scoring to evaluate how well the prompt performs.
+#     """
+#     # Interpolate the prompt template with the golden's input
+#     final_prompt = prompt.interpolate(query=golden.input)
+
+#     # Call your LLM
+#     response = client.chat.completions.create(
+#         model="gpt-4o-mini",
+#         messages=[{"role": "user", "content": final_prompt}],
+#     )
+
+#     return response.choices[0].message.content
+
+
+# # Define your initial prompt template (intentionally bad for testing optimization)
+# prompt = Prompt(
+#     text_template="""idk maybe try to respond to this thing if u want lol
+
+# {query}
+
+# whatever:"""
+# )
+
+# # Define your evaluation dataset (goldens)
+# goldens = [
+#     Golden(
+#         input="What is the capital of France?",
+#         expected_output="Paris",
+#     ),
+#     Golden(
+#         input="Who wrote Romeo and Juliet?",
+#         expected_output="William Shakespeare",
+#     ),
+#     Golden(
+#         input="What is the chemical symbol for gold?",
+#         expected_output="Au",
+#     ),
+#     Golden(
+#         input="In what year did World War II end?",
+#         expected_output="1945",
+#     ),
+# ]
+
+# # Define metrics to optimize for
+# metrics = [AnswerRelevancyMetric(threshold=0.7)]
+
+# from deepeval.optimizer.configs import DisplayConfig
+# from deepeval.optimizer.algorithms import GEPA
+
+# # Create the optimizer
+# optimizer = PromptOptimizer(
+#     model_callback=model_callback,
+#     metrics=metrics,
+#     optimizer_model="gpt-4o",  # Model used for rewriting prompts
+#     display_config=DisplayConfig(announce_ties=True),
+#     algorithm=GEPA(iterations=1),
+# )
+
+# # Run optimization
+# optimized_prompt = optimizer.optimize(prompt=prompt, goldens=goldens)
+
+# # Print results
+# print("\n" + "=" * 60)
+# print("OPTIMIZATION COMPLETE")
+# print("=" * 60)
+# print(f"\nOriginal prompt:\n{prompt.text_template}")
+# print(f"\nOptimized prompt:\n{optimized_prompt.text_template}")

deepeval/test_case/__init__.py

@@ -4,6 +4,7 @@
     ToolCall,
     ToolCallParams,
     MLLMImage,
+    Context,
 )
 from .conversational_test_case import (
     ConversationalTestCase,
@@ -24,6 +25,7 @@
     "LLMTestCaseParams",
     "ToolCall",
     "ToolCallParams",
+    "Context",
     "ConversationalTestCase",
     "Turn",
     "TurnParams",

deepeval/test_case/conversational_test_case.py

@@ -6,7 +6,7 @@
     model_validator,
     AliasChoices,
 )
-from typing import List, Optional, Dict, Literal
+from typing import List, Optional, Dict, Literal, Union
 from copy import deepcopy
 from enum import Enum
 
@@ -18,7 +18,7 @@
     MCPToolCall,
     validate_mcp_servers,
 )
-from deepeval.test_case.llm_test_case import _MLLM_IMAGE_REGISTRY
+from deepeval.test_case.llm_test_case import _MLLM_IMAGE_REGISTRY, Context
 
 
 class TurnParams(Enum):
@@ -131,7 +131,7 @@ def validate_input(cls, data):
 class ConversationalTestCase(BaseModel):
     turns: List[Turn]
     scenario: Optional[str] = Field(default=None)
-    context: Optional[List[str]] = Field(default=None)
+    context: Optional[List[Union[str, Context]]] = Field(default=None)
     name: Optional[str] = Field(default=None)
     user_description: Optional[str] = Field(
         default=None,
@@ -163,9 +163,41 @@ class ConversationalTestCase(BaseModel):
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
+    _context_items: Optional[List[Union[str, Context]]] = PrivateAttr(
+        default=None
+    )
 
     @model_validator(mode="after")
-    def set_is_multimodal(self):
+    def post_init(self):
+
+        self._handle_context_data()
+        self._set_is_multimodal()
+
+        return self
+
+    def _handle_context_data(self):
+        if self.context is None:
+            return
+
+        self._context_items = self.context[:]
+
+        resolved_context = []
+
+        for item in self.context:
+            if isinstance(item, Context):
+                resolved = item.resolve_contexts()
+                if isinstance(resolved, list):
+                    resolved_context.extend(resolved)
+                else:
+                    resolved_context.append(resolved)
+            else:
+                resolved_context.append(item)
+
+        self.context = resolved_context
+
+        return self
+
+    def _set_is_multimodal(self):
         import re
 
         if self.multimodal is True:
@@ -195,8 +227,6 @@ def set_is_multimodal(self):
                         for context in turn.retrieval_context
                     )
 
-        return self
-
     @model_validator(mode="before")
     def validate_input(cls, data):
         turns = data.get("turns")
@@ -209,9 +239,12 @@ def validate_input(cls, data):
         # Ensure `context` is None or a list of strings
         if context is not None:
             if not isinstance(context, list) or not all(
-                isinstance(item, str) for item in context
+                (isinstance(item, str) or isinstance(item, Context))
+                for item in context
             ):
-                raise TypeError("'context' must be None or a list of strings")
+                raise TypeError(
+                    "'context' must be None or a list of  or 'Context'"
+                )
 
         if mcp_servers is not None:
             validate_mcp_servers(mcp_servers)