Update evaluator.py

Author: codelion

openevolve/evaluator.py

@@ -314,10 +314,6 @@ async def _cascade_evaluate(
 
         Returns:
             Dictionary of metrics or EvaluationResult with metrics and artifacts
-
-        Raises:
-            asyncio.TimeoutError: If any stage exceeds timeout
-            Exception: If any evaluation stage raises an exception
         """
         # Import the evaluation module to get cascade functions if they exist
         try:
@@ -339,12 +335,34 @@ async def _cascade_evaluate(
                 return await self._direct_evaluate(program_path)
 
             # Run first stage with timeout
-            async def run_stage1():
-                loop = asyncio.get_event_loop()
-                return await loop.run_in_executor(None, module.evaluate_stage1, program_path)
+            try:
 
-            stage1_result = await asyncio.wait_for(run_stage1(), timeout=self.config.timeout)
-            stage1_eval_result = self._process_evaluation_result(stage1_result)
+                async def run_stage1():
+                    loop = asyncio.get_event_loop()
+                    return await loop.run_in_executor(None, module.evaluate_stage1, program_path)
+
+                stage1_result = await asyncio.wait_for(run_stage1(), timeout=self.config.timeout)
+                stage1_eval_result = self._process_evaluation_result(stage1_result)
+            except asyncio.TimeoutError:
+                logger.warning(f"Stage 1 evaluation timed out after {self.config.timeout}s")
+                return EvaluationResult(
+                    metrics={"stage1_passed": 0.0, "error": 0.0, "timeout": True},
+                    artifacts={
+                        "failure_stage": "stage1",
+                        "timeout": True,
+                    },
+                )
+            except Exception as e:
+                logger.error(f"Error in stage 1 evaluation: {str(e)}")
+                # Capture stage 1 failure as artifacts
+                return EvaluationResult(
+                    metrics={"stage1_passed": 0.0, "error": 0.0},
+                    artifacts={
+                        "stderr": str(e),
+                        "traceback": traceback.format_exc(),
+                        "failure_stage": "stage1",
+                    },
+                )
 
             # Check threshold
             if not self._passes_threshold(
@@ -357,12 +375,38 @@ async def run_stage1():
                 return stage1_eval_result
 
             # Run second stage with timeout
-            async def run_stage2():
-                loop = asyncio.get_event_loop()
-                return await loop.run_in_executor(None, module.evaluate_stage2, program_path)
+            try:
 
-            stage2_result = await asyncio.wait_for(run_stage2(), timeout=self.config.timeout)
-            stage2_eval_result = self._process_evaluation_result(stage2_result)
+                async def run_stage2():
+                    loop = asyncio.get_event_loop()
+                    return await loop.run_in_executor(None, module.evaluate_stage2, program_path)
+
+                stage2_result = await asyncio.wait_for(run_stage2(), timeout=self.config.timeout)
+                stage2_eval_result = self._process_evaluation_result(stage2_result)
+            except asyncio.TimeoutError:
+                logger.warning(f"Stage 2 evaluation timed out after {self.config.timeout}s")
+                # Capture stage 2 failure, but keep stage 1 results
+                stage1_eval_result.artifacts.update(
+                    {
+                        "stage2_timeout": True,
+                        "failure_stage": "stage2",
+                    }
+                )
+                stage1_eval_result.metrics["stage2_passed"] = 0.0
+                stage1_eval_result.metrics["timeout"] = True
+                return stage1_eval_result
+            except Exception as e:
+                logger.error(f"Error in stage 2 evaluation: {str(e)}")
+                # Capture stage 2 failure, but keep stage 1 results
+                stage1_eval_result.artifacts.update(
+                    {
+                        "stage2_stderr": str(e),
+                        "stage2_traceback": traceback.format_exc(),
+                        "failure_stage": "stage2",
+                    }
+                )
+                stage1_eval_result.metrics["stage2_passed"] = 0.0
+                return stage1_eval_result
 
             # Merge results from stage 1 and 2
             merged_metrics = {}
@@ -393,12 +437,38 @@ async def run_stage2():
                 return merged_result
 
             # Run third stage with timeout
-            async def run_stage3():
-                loop = asyncio.get_event_loop()
-                return await loop.run_in_executor(None, module.evaluate_stage3, program_path)
+            try:
 
-            stage3_result = await asyncio.wait_for(run_stage3(), timeout=self.config.timeout)
-            stage3_eval_result = self._process_evaluation_result(stage3_result)
+                async def run_stage3():
+                    loop = asyncio.get_event_loop()
+                    return await loop.run_in_executor(None, module.evaluate_stage3, program_path)
+
+                stage3_result = await asyncio.wait_for(run_stage3(), timeout=self.config.timeout)
+                stage3_eval_result = self._process_evaluation_result(stage3_result)
+            except asyncio.TimeoutError:
+                logger.warning(f"Stage 3 evaluation timed out after {self.config.timeout}s")
+                # Capture stage 3 failure, but keep previous results
+                merged_result.artifacts.update(
+                    {
+                        "stage3_timeout": True,
+                        "failure_stage": "stage3",
+                    }
+                )
+                merged_result.metrics["stage3_passed"] = 0.0
+                merged_result.metrics["timeout"] = True
+                return merged_result
+            except Exception as e:
+                logger.error(f"Error in stage 3 evaluation: {str(e)}")
+                # Capture stage 3 failure, but keep previous results
+                merged_result.artifacts.update(
+                    {
+                        "stage3_stderr": str(e),
+                        "stage3_traceback": traceback.format_exc(),
+                        "failure_stage": "stage3",
+                    }
+                )
+                merged_result.metrics["stage3_passed"] = 0.0
+                return merged_result
 
             # Merge stage 3 results
             for name, value in stage3_eval_result.metrics.items():
@@ -411,8 +481,15 @@ async def run_stage3():
 
         except Exception as e:
             logger.error(f"Error in cascade evaluation: {str(e)}")
-            # Re-raise the exception to allow retry handling at higher level
-            raise
+            # Return proper cascade failure result instead of re-raising
+            return EvaluationResult(
+                metrics={"stage1_passed": 0.0, "error": 0.0},
+                artifacts={
+                    "stderr": str(e),
+                    "traceback": traceback.format_exc(),
+                    "failure_stage": "cascade_setup",
+                },
+            )
 
     async def _llm_evaluate(self, program_code: str, program_id: str = "") -> Dict[str, float]:
         """