fix(evaluate): evaluate function add evalset (#204)

Author: floritange

veadk/evaluation/adk_evaluator/adk_evaluator.py

@@ -24,6 +24,8 @@
 )
 from google.adk.evaluation.eval_case import IntermediateData, Invocation
 from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.eval_set import EvalSet
+from typing import Optional
 from typing_extensions import override
 from veadk.evaluation.base_evaluator import BaseEvaluator
 from types import SimpleNamespace
@@ -52,7 +54,8 @@ def __init__(
     @override
     async def evaluate(
         self,
-        eval_set_file_path: str,
+        eval_set: Optional[EvalSet] = None,
+        eval_set_file_path: Optional[str] = None,
         eval_id: str = f"test_{formatted_timestamp()}",
         tool_score_threshold: float = 1.0,
         response_match_score_threshold: float = 0.8,
@@ -104,7 +107,7 @@ async def evaluate(
         # Iterate each test file and evaluate per-case, per-metric
         for test_file in test_files:
             # Build in-memory evaluation cases via BaseEvaluator from the provided file
-            self.build_eval_set(test_file)
+            self.build_eval_set(eval_set, test_file)
 
             evaluation_result_list = []
 

veadk/evaluation/base_evaluator.py

@@ -17,7 +17,7 @@
 import time
 import uuid
 from abc import abstractmethod
-from typing import Any
+from typing import Any, Optional
 
 from google.adk import Runner
 from google.adk.evaluation.eval_set import EvalSet
@@ -210,33 +210,43 @@ def _build_eval_set_from_tracing_json(self, tracing_json_path: str) -> EvalSet:
 
         return evalset
 
-    def build_eval_set(self, file_path: str):
+    def build_eval_set(
+        self, eval_set: Optional[EvalSet] = None, file_path: Optional[str] = None
+    ):
         """Generate evaluation data from a given file and assign it to the class attribute `invocation_list`."""
-        eval_case_data_list: list[EvalTestCase] = []
 
-        try:
-            with open(file_path, "r", encoding="utf-8") as f:
-                file_content = json.load(f)
-        except json.JSONDecodeError as e:
-            raise ValueError(f"Invalid JSON format in file {file_path}: {e}")
-        except Exception as e:
-            raise ValueError(f"Error reading file {file_path}: {e}")
-
-        if isinstance(file_content, dict) and "eval_cases" in file_content:
-            eval_cases = self._build_eval_set_from_eval_json(file_path).eval_cases
-        elif (
-            isinstance(file_content, list)
-            and len(file_content) > 0
-            and all(
-                isinstance(span, dict) and "trace_id" in span for span in file_content
-            )
-        ):
-            eval_cases = self._build_eval_set_from_tracing_json(file_path).eval_cases
+        if eval_set is None and file_path is None:
+            raise ValueError("eval_set or file_path is required")
+        if eval_set:
+            eval_cases = eval_set.eval_cases
         else:
-            raise ValueError(
-                f"Unsupported file format in {file_path}. Please provide a valid file."
-            )
+            try:
+                with open(file_path, "r", encoding="utf-8") as f:
+                    file_content = json.load(f)
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Invalid JSON format in file {file_path}: {e}")
+            except Exception as e:
+                raise ValueError(f"Error reading file {file_path}: {e}")
+
+            if isinstance(file_content, dict) and "eval_cases" in file_content:
+                eval_cases = self._build_eval_set_from_eval_json(file_path).eval_cases
+            elif (
+                isinstance(file_content, list)
+                and len(file_content) > 0
+                and all(
+                    isinstance(span, dict) and "trace_id" in span
+                    for span in file_content
+                )
+            ):
+                eval_cases = self._build_eval_set_from_tracing_json(
+                    file_path
+                ).eval_cases
+            else:
+                raise ValueError(
+                    f"Unsupported file format in {file_path}. Please provide a valid file."
+                )
 
+        eval_case_data_list: list[EvalTestCase] = []
         for eval_case in eval_cases:
             eval_case_data = EvalTestCase(invocations=[])
             if eval_case.session_input:
@@ -384,8 +394,9 @@ def get_eval_set_information(self) -> list[list[dict[str, Any]]]:
     @abstractmethod
     async def evaluate(
         self,
-        eval_set_file_path: str,
         metrics: list[Any],
+        eval_set: Optional[EvalSet],
+        eval_set_file_path: Optional[str],
         eval_id: str,
     ):
         """An abstract method for evaluation based on metrics。"""

veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py

@@ -22,7 +22,8 @@
 from deepeval.test_case import LLMTestCase
 from deepeval.test_case.llm_test_case import ToolCall
 from typing_extensions import override
-
+from typing import Optional
+from google.adk.evaluation.eval_set import EvalSet
 from veadk.config import getenv
 from veadk.evaluation.base_evaluator import BaseEvaluator, EvalResultData, MetricResult
 from veadk.evaluation.types import EvalResultCaseData, EvalResultMetadata
@@ -77,13 +78,14 @@ def __init__(
     @override
     async def evaluate(
         self,
-        eval_set_file_path: str,
         metrics: list[BaseMetric],
+        eval_set: Optional[EvalSet] = None,
+        eval_set_file_path: Optional[str] = None,
         eval_id: str = f"test_{formatted_timestamp()}",
     ):
         """Target to Google ADK, we will use the same evaluation case format as Google ADK."""
         # Get evaluation data by parsing eval set file
-        self.build_eval_set(eval_set_file_path)
+        self.build_eval_set(eval_set, eval_set_file_path)
 
         # Get actual data by running agent
         logger.info("Start to run agent for actual data.")