chore: resolve all ruff and mypy issues in experimental module

resolve linting, typing, and import issues; update Okta test

Author: greysonlalonde

src/crewai/experimental/__init__.py

@@ -1,40 +1,39 @@
 from crewai.experimental.evaluation import (
+    AgentEvaluationResult,
+    AgentEvaluator,
     BaseEvaluator,
     EvaluationScore,
-    MetricCategory,
-    AgentEvaluationResult,
-    SemanticQualityEvaluator,
+    EvaluationTraceCallback,
+    ExperimentResult,
+    ExperimentResults,
+    ExperimentRunner,
     GoalAlignmentEvaluator,
-    ReasoningEfficiencyEvaluator,
-    ToolSelectionEvaluator,
+    MetricCategory,
     ParameterExtractionEvaluator,
+    ReasoningEfficiencyEvaluator,
+    SemanticQualityEvaluator,
     ToolInvocationEvaluator,
-    EvaluationTraceCallback,
-    create_evaluation_callbacks,
-    AgentEvaluator,
+    ToolSelectionEvaluator,
     create_default_evaluator,
-    ExperimentRunner,
-    ExperimentResults,
-    ExperimentResult,
+    create_evaluation_callbacks,
 )
 
-
 __all__ = [
+    "AgentEvaluationResult",
+    "AgentEvaluator",
     "BaseEvaluator",
     "EvaluationScore",
-    "MetricCategory",
-    "AgentEvaluationResult",
-    "SemanticQualityEvaluator",
+    "EvaluationTraceCallback",
+    "ExperimentResult",
+    "ExperimentResults",
+    "ExperimentRunner",
     "GoalAlignmentEvaluator",
-    "ReasoningEfficiencyEvaluator",
-    "ToolSelectionEvaluator",
+    "MetricCategory",
     "ParameterExtractionEvaluator",
+    "ReasoningEfficiencyEvaluator",
+    "SemanticQualityEvaluator",
     "ToolInvocationEvaluator",
-    "EvaluationTraceCallback",
-    "create_evaluation_callbacks",
-    "AgentEvaluator",
+    "ToolSelectionEvaluator",
     "create_default_evaluator",
-    "ExperimentRunner",
-    "ExperimentResults",
-    "ExperimentResult"
-]
+    "create_evaluation_callbacks",
+]

src/crewai/experimental/evaluation/__init__.py

@@ -1,51 +1,47 @@
+from crewai.experimental.evaluation.agent_evaluator import (
+    AgentEvaluator,
+    create_default_evaluator,
+)
 from crewai.experimental.evaluation.base_evaluator import (
+    AgentEvaluationResult,
     BaseEvaluator,
     EvaluationScore,
     MetricCategory,
-    AgentEvaluationResult
-)
-
-from crewai.experimental.evaluation.metrics import (
-    SemanticQualityEvaluator,
-    GoalAlignmentEvaluator,
-    ReasoningEfficiencyEvaluator,
-    ToolSelectionEvaluator,
-    ParameterExtractionEvaluator,
-    ToolInvocationEvaluator
 )
-
 from crewai.experimental.evaluation.evaluation_listener import (
     EvaluationTraceCallback,
-    create_evaluation_callbacks
+    create_evaluation_callbacks,
 )
-
-from crewai.experimental.evaluation.agent_evaluator import (
-    AgentEvaluator,
-    create_default_evaluator
-)
-
 from crewai.experimental.evaluation.experiment import (
-    ExperimentRunner,
+    ExperimentResult,
     ExperimentResults,
-    ExperimentResult
+    ExperimentRunner,
+)
+from crewai.experimental.evaluation.metrics import (
+    GoalAlignmentEvaluator,
+    ParameterExtractionEvaluator,
+    ReasoningEfficiencyEvaluator,
+    SemanticQualityEvaluator,
+    ToolInvocationEvaluator,
+    ToolSelectionEvaluator,
 )
 
 __all__ = [
+    "AgentEvaluationResult",
+    "AgentEvaluator",
     "BaseEvaluator",
     "EvaluationScore",
-    "MetricCategory",
-    "AgentEvaluationResult",
-    "SemanticQualityEvaluator",
+    "EvaluationTraceCallback",
+    "ExperimentResult",
+    "ExperimentResults",
+    "ExperimentRunner",
     "GoalAlignmentEvaluator",
-    "ReasoningEfficiencyEvaluator",
-    "ToolSelectionEvaluator",
+    "MetricCategory",
     "ParameterExtractionEvaluator",
+    "ReasoningEfficiencyEvaluator",
+    "SemanticQualityEvaluator",
     "ToolInvocationEvaluator",
-    "EvaluationTraceCallback",
-    "create_evaluation_callbacks",
-    "AgentEvaluator",
+    "ToolSelectionEvaluator",
     "create_default_evaluator",
-    "ExperimentRunner",
-    "ExperimentResults",
-    "ExperimentResult"
+    "create_evaluation_callbacks",
 ]

src/crewai/experimental/evaluation/agent_evaluator.py

@@ -1,34 +1,36 @@
 import threading
-from typing import Any, Optional
+from collections.abc import Sequence
+from typing import Any
 
-from crewai.experimental.evaluation.base_evaluator import (
-    AgentEvaluationResult,
-    AggregationStrategy,
-)
 from crewai.agent import Agent
-from crewai.task import Task
-from crewai.experimental.evaluation.evaluation_display import EvaluationDisplayFormatter
+from crewai.agents.agent_builder.base_agent import BaseAgent
+from crewai.events.event_bus import crewai_event_bus
 from crewai.events.types.agent_events import (
-    AgentEvaluationStartedEvent,
     AgentEvaluationCompletedEvent,
     AgentEvaluationFailedEvent,
+    AgentEvaluationStartedEvent,
+    LiteAgentExecutionCompletedEvent,
 )
-from crewai.experimental.evaluation import BaseEvaluator, create_evaluation_callbacks
-from collections.abc import Sequence
-from crewai.events.event_bus import crewai_event_bus
-from crewai.events.utils.console_formatter import ConsoleFormatter
 from crewai.events.types.task_events import TaskCompletedEvent
-from crewai.events.types.agent_events import LiteAgentExecutionCompletedEvent
+from crewai.events.utils.console_formatter import ConsoleFormatter
 from crewai.experimental.evaluation.base_evaluator import (
     AgentAggregatedEvaluationResult,
+    AgentEvaluationResult,
+    AggregationStrategy,
+    BaseEvaluator,
     EvaluationScore,
     MetricCategory,
 )
+from crewai.experimental.evaluation.evaluation_display import EvaluationDisplayFormatter
+from crewai.experimental.evaluation.evaluation_listener import (
+    create_evaluation_callbacks,
+)
+from crewai.task import Task
 
 
 class ExecutionState:
-    current_agent_id: Optional[str] = None
-    current_task_id: Optional[str] = None
+    current_agent_id: str | None = None
+    current_task_id: str | None = None
 
     def __init__(self):
         self.traces = {}
@@ -40,10 +42,10 @@ def __init__(self):
 class AgentEvaluator:
     def __init__(
         self,
-        agents: list[Agent],
+        agents: list[Agent] | list[BaseAgent],
         evaluators: Sequence[BaseEvaluator] | None = None,
     ):
-        self.agents: list[Agent] = agents
+        self.agents: list[Agent] | list[BaseAgent] = agents
         self.evaluators: Sequence[BaseEvaluator] | None = evaluators
 
         self.callback = create_evaluation_callbacks()
@@ -75,7 +77,8 @@ def _subscribe_to_events(self) -> None:
         )
 
     def _handle_task_completed(self, source: Any, event: TaskCompletedEvent) -> None:
-        assert event.task is not None
+        if event.task is None:
+            raise ValueError("TaskCompletedEvent must have a task")
         agent = event.task.agent
         if (
             agent
@@ -92,9 +95,8 @@ def _handle_task_completed(self, source: Any, event: TaskCompletedEvent) -> None
             state.current_agent_id = str(agent.id)
             state.current_task_id = str(event.task.id)
 
-            assert (
-                state.current_agent_id is not None and state.current_task_id is not None
-            )
+            if state.current_agent_id is None or state.current_task_id is None:
+                raise ValueError("Agent ID and Task ID must not be None")
             trace = self.callback.get_trace(
                 state.current_agent_id, state.current_task_id
             )
@@ -146,9 +148,8 @@ def _handle_lite_agent_completed(
             if not target_agent:
                 return
 
-            assert (
-                state.current_agent_id is not None and state.current_task_id is not None
-            )
+            if state.current_agent_id is None or state.current_task_id is None:
+                raise ValueError("Agent ID and Task ID must not be None")
             trace = self.callback.get_trace(
                 state.current_agent_id, state.current_task_id
             )
@@ -244,7 +245,7 @@ def display_evaluation_with_feedback(self) -> None:
 
     def evaluate(
         self,
-        agent: Agent,
+        agent: Agent | BaseAgent,
         execution_trace: dict[str, Any],
         final_output: Any,
         state: ExecutionState,
@@ -255,7 +256,8 @@ def evaluate(
             task_id=state.current_task_id or (str(task.id) if task else "unknown_task"),
         )
 
-        assert self.evaluators is not None
+        if self.evaluators is None:
+            raise ValueError("Evaluators must be initialized")
         task_id = str(task.id) if task else None
         for evaluator in self.evaluators:
             try:
@@ -276,15 +278,15 @@ def evaluate(
                     metric_category=evaluator.metric_category,
                     score=score,
                 )
-            except Exception as e:
+            except Exception as e:  # noqa: PERF203
                 self.emit_evaluation_failed_event(
                     agent_role=agent.role,
                     agent_id=str(agent.id),
                     task_id=task_id,
                     error=str(e),
                 )
                 self.console_formatter.print(
-                    f"Error in {evaluator.metric_category.value} evaluator: {str(e)}"
+                    f"Error in {evaluator.metric_category.value} evaluator: {e!s}"
                 )
 
         return result
@@ -337,14 +339,14 @@ def emit_evaluation_failed_event(
         )
 
 
-def create_default_evaluator(agents: list[Agent], llm: None = None):
+def create_default_evaluator(agents: list[Agent] | list[BaseAgent], llm: None = None):
     from crewai.experimental.evaluation import (
         GoalAlignmentEvaluator,
-        SemanticQualityEvaluator,
-        ToolSelectionEvaluator,
         ParameterExtractionEvaluator,
-        ToolInvocationEvaluator,
         ReasoningEfficiencyEvaluator,
+        SemanticQualityEvaluator,
+        ToolInvocationEvaluator,
+        ToolSelectionEvaluator,
     )
 
     evaluators = [