Update test_evaluator_timeout.py

Author: codelion

tests/test_evaluator_timeout.py

@@ -21,7 +21,7 @@ def setUp(self):
         # Create a test evaluation file
         self.test_eval_file = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False)
 
-        # Write test evaluation functions
+        # Write test evaluation functions with shorter sleep times for faster tests
         self.test_eval_file.write(
             """
 import time
@@ -32,12 +32,12 @@ def evaluate(program_path):
         code = f.read()
     
     if 'SLEEP_LONG' in code:
-        # Sleep for a long time to trigger timeout
-        time.sleep(30)
+        # Sleep for a long time to trigger timeout (reduced for faster tests)
+        time.sleep(8)
         return {"score": 1.0}
     elif 'SLEEP_SHORT' in code:
         # Sleep for a short time that should not timeout
-        time.sleep(1)
+        time.sleep(0.5)
         return {"score": 0.8}
     elif 'RAISE_ERROR' in code:
         # Raise an error to trigger retries
@@ -51,7 +51,7 @@ def evaluate_stage1(program_path):
         code = f.read()
     
     if 'STAGE1_TIMEOUT' in code:
-        time.sleep(30)
+        time.sleep(8)
         return {"stage1_score": 1.0}
     else:
         return {"stage1_score": 0.7}
@@ -61,7 +61,7 @@ def evaluate_stage2(program_path):
         code = f.read()
     
     if 'STAGE2_TIMEOUT' in code:
-        time.sleep(30)
+        time.sleep(8)
         return {"stage2_score": 1.0}
     else:
         return {"stage2_score": 0.8}
@@ -71,7 +71,7 @@ def evaluate_stage3(program_path):
         code = f.read()
     
     if 'STAGE3_TIMEOUT' in code:
-        time.sleep(30)
+        time.sleep(8)
         return {"stage3_score": 1.0}
     else:
         return {"stage3_score": 0.9}
@@ -84,8 +84,8 @@ def tearDown(self):
         if os.path.exists(self.test_eval_file.name):
             os.unlink(self.test_eval_file.name)
 
-    def _create_evaluator(self, timeout=5, cascade_evaluation=False):
-        """Helper to create evaluator with given settings"""
+    def _create_evaluator(self, timeout=3, cascade_evaluation=False):
+        """Helper to create evaluator with given settings (shorter timeout for faster tests)"""
         config = EvaluatorConfig()
         config.timeout = timeout
         config.max_retries = 1  # Minimal retries for faster testing
@@ -103,7 +103,7 @@ def test_fast_evaluation_completes(self):
         """Test that fast evaluations complete successfully"""
 
         async def run_test():
-            evaluator = self._create_evaluator(timeout=5)
+            evaluator = self._create_evaluator(timeout=3)
             program_code = "def test(): return 'fast'"
             start_time = time.time()
 
@@ -112,7 +112,7 @@ async def run_test():
             elapsed_time = time.time() - start_time
 
             # Should complete quickly
-            self.assertLess(elapsed_time, 3.0)
+            self.assertLess(elapsed_time, 2.0)
             # Should return successful result
             self.assertIn("score", result)
             self.assertEqual(result["score"], 0.5)
@@ -126,7 +126,7 @@ def test_short_evaluation_completes(self):
         """Test that evaluations shorter than timeout complete successfully"""
 
         async def run_test():
-            evaluator = self._create_evaluator(timeout=5)
+            evaluator = self._create_evaluator(timeout=3)
             program_code = "# SLEEP_SHORT\ndef test(): return 'short'"
             start_time = time.time()
 
@@ -135,7 +135,7 @@ async def run_test():
             elapsed_time = time.time() - start_time
 
             # Should complete within timeout
-            self.assertLess(elapsed_time, 5)
+            self.assertLess(elapsed_time, 3)
             # Should return successful result
             self.assertIn("score", result)
             self.assertEqual(result["score"], 0.8)
@@ -149,7 +149,7 @@ def test_long_evaluation_times_out(self):
         """Test that long evaluations time out properly"""
 
         async def run_test():
-            evaluator = self._create_evaluator(timeout=5)
+            evaluator = self._create_evaluator(timeout=3)
             program_code = "# SLEEP_LONG\ndef test(): return 'long'"
             start_time = time.time()
 
@@ -158,8 +158,8 @@ async def run_test():
             elapsed_time = time.time() - start_time
 
             # Should complete around the timeout period (allowing some margin)
-            self.assertGreater(elapsed_time, 4)
-            self.assertLess(elapsed_time, 8)
+            self.assertGreater(elapsed_time, 2.5)
+            self.assertLess(elapsed_time, 5)
 
             # Should return timeout result
             self.assertIn("error", result)
@@ -173,7 +173,7 @@ def test_cascade_evaluation_timeout_stage1(self):
         """Test timeout in cascade evaluation stage 1"""
 
         async def run_test():
-            evaluator = self._create_evaluator(timeout=5, cascade_evaluation=True)
+            evaluator = self._create_evaluator(timeout=3, cascade_evaluation=True)
             program_code = "# STAGE1_TIMEOUT\ndef test(): return 'stage1_timeout'"
             start_time = time.time()
 
@@ -182,8 +182,8 @@ async def run_test():
             elapsed_time = time.time() - start_time
 
             # Should timeout around the configured timeout
-            self.assertGreater(elapsed_time, 4)
-            self.assertLess(elapsed_time, 8)
+            self.assertGreater(elapsed_time, 2.5)
+            self.assertLess(elapsed_time, 5)
 
             # Should return stage1 timeout result
             self.assertIn("stage1_passed", result)
@@ -197,7 +197,7 @@ def test_cascade_evaluation_timeout_stage2(self):
         """Test timeout in cascade evaluation stage 2"""
 
         async def run_test():
-            evaluator = self._create_evaluator(timeout=5, cascade_evaluation=True)
+            evaluator = self._create_evaluator(timeout=3, cascade_evaluation=True)
             program_code = "# STAGE2_TIMEOUT\ndef test(): return 'stage2_timeout'"
             start_time = time.time()
 
@@ -206,8 +206,8 @@ async def run_test():
             elapsed_time = time.time() - start_time
 
             # Should timeout on stage 2, but stage 1 should complete first
-            self.assertGreater(elapsed_time, 4)
-            self.assertLess(elapsed_time, 8)
+            self.assertGreater(elapsed_time, 2.5)
+            self.assertLess(elapsed_time, 5)
 
             # Should have stage1 result but stage2 timeout
             self.assertIn("stage1_score", result)
@@ -223,7 +223,7 @@ def test_cascade_evaluation_timeout_stage3(self):
         """Test timeout in cascade evaluation stage 3"""
 
         async def run_test():
-            evaluator = self._create_evaluator(timeout=5, cascade_evaluation=True)
+            evaluator = self._create_evaluator(timeout=3, cascade_evaluation=True)
             program_code = "# STAGE3_TIMEOUT\ndef test(): return 'stage3_timeout'"
             start_time = time.time()
 
@@ -232,8 +232,8 @@ async def run_test():
             elapsed_time = time.time() - start_time
 
             # Should timeout on stage 3, but stages 1 and 2 should complete first
-            self.assertGreater(elapsed_time, 4)
-            self.assertLess(elapsed_time, 8)
+            self.assertGreater(elapsed_time, 2.5)
+            self.assertLess(elapsed_time, 5)
 
             # Should have stage1 and stage2 results but stage3 timeout
             self.assertIn("stage1_score", result)
@@ -252,7 +252,7 @@ def test_timeout_config_respected(self):
 
         async def run_test():
             # Create evaluator with different timeout
-            evaluator = self._create_evaluator(timeout=10)
+            evaluator = self._create_evaluator(timeout=5)
 
             program_code = "# SLEEP_LONG\ndef test(): return 'long'"
             start_time = time.time()
@@ -261,9 +261,9 @@ async def run_test():
 
             elapsed_time = time.time() - start_time
 
-            # Should timeout around 10 seconds, not 5
-            self.assertGreater(elapsed_time, 9)
-            self.assertLess(elapsed_time, 13)
+            # Should timeout around 5 seconds, not 3
+            self.assertGreater(elapsed_time, 4.5)
+            self.assertLess(elapsed_time, 7)
 
             # Should return timeout result
             self.assertIn("timeout", result)
@@ -277,7 +277,7 @@ def test_multiple_retries_with_errors(self):
         async def run_test():
             # Create evaluator with more retries
             config = EvaluatorConfig()
-            config.timeout = 10  # Long timeout to avoid timeout during this test
+            config.timeout = 8  # Long timeout to avoid timeout during this test
             config.max_retries = 2  # 3 total attempts
             config.cascade_evaluation = False
 
@@ -300,7 +300,7 @@ async def run_test():
             # Each attempt should fail quickly, plus 1 second sleep between retries
             # So total time should be around 2-3 seconds (quick failures + 2 sleep periods)
             self.assertGreater(elapsed_time, 1.8)  # At least 2 sleep periods
-            self.assertLess(elapsed_time, 5)  # But not too long
+            self.assertLess(elapsed_time, 4)  # But not too long
 
             # Should return error result after all retries fail
             self.assertIn("error", result)
@@ -314,7 +314,7 @@ def test_timeout_does_not_trigger_retries(self):
         async def run_test():
             # Create evaluator with retries enabled
             config = EvaluatorConfig()
-            config.timeout = 3  # Short timeout
+            config.timeout = 2  # Short timeout
             config.max_retries = 2  # Would allow 3 attempts if retries were triggered
             config.cascade_evaluation = False
 
@@ -333,10 +333,10 @@ async def run_test():
 
             elapsed_time = time.time() - start_time
 
-            # Should timeout only once (~3 seconds), not retry multiple times
-            # If retries were happening, this would take ~9 seconds
-            self.assertGreater(elapsed_time, 2.5)  # At least the timeout period
-            self.assertLess(elapsed_time, 5)  # But not multiple timeout periods
+            # Should timeout only once (~2 seconds), not retry multiple times
+            # If retries were happening, this would take ~6 seconds
+            self.assertGreater(elapsed_time, 1.8)  # At least the timeout period
+            self.assertLess(elapsed_time, 3.5)  # But not multiple timeout periods
 
             # Should return timeout result
             self.assertIn("timeout", result)
@@ -350,7 +350,7 @@ def test_artifacts_on_timeout(self):
         async def run_test():
             # Enable artifacts
             with patch.dict(os.environ, {"ENABLE_ARTIFACTS": "true"}):
-                evaluator = self._create_evaluator(timeout=5)
+                evaluator = self._create_evaluator(timeout=3)
                 program_code = "# SLEEP_LONG\ndef test(): return 'long'"
 
                 # Execute evaluation
@@ -382,7 +382,7 @@ async def run_test():
                     "timeout_duration", artifacts, "Artifacts should contain timeout_duration"
                 )
                 self.assertEqual(
-                    artifacts["timeout_duration"], 5, "timeout_duration should match config"
+                    artifacts["timeout_duration"], 3, "timeout_duration should match config"
                 )
 
                 print(f"✅ Artifacts captured correctly: {list(artifacts.keys())}")
@@ -405,8 +405,8 @@ async def run_test():
 import time
 
 def evaluate(program_path):
-    # Simulate a very long evaluation (like the 11-hour case)
-    time.sleep(20)  # 20 seconds to test timeout
+    # Simulate a very long evaluation (like the 11-hour case) 
+    time.sleep(6)  # 6 seconds to test timeout (reduced for faster tests)
     return {"accReturn": 0.1, "CalmarRatio": 0.9, "combined_score": 0.82}
 """
             )
@@ -415,7 +415,7 @@ def evaluate(program_path):
             try:
                 # Configure like user's config but with shorter timeout for testing
                 config = EvaluatorConfig()
-                config.timeout = 5  # 5 seconds instead of 600
+                config.timeout = 3  # 3 seconds instead of 600
                 config.max_retries = 1
                 config.cascade_evaluation = False
                 config.parallel_evaluations = 1
@@ -438,9 +438,9 @@ def search_algorithm():
                 result = await evaluator.evaluate_program(program_code, "financial_test")
                 elapsed_time = time.time() - start_time
 
-                # Should timeout in ~5 seconds, not 20+ seconds
-                self.assertLess(elapsed_time, 8)
-                self.assertGreater(elapsed_time, 4)
+                # Should timeout in ~3 seconds, not 6+ seconds
+                self.assertLess(elapsed_time, 5)
+                self.assertGreater(elapsed_time, 2.5)
 
                 # Should return timeout error
                 self.assertIn("error", result)