Fix dynamic timeout

tests/explorer/scheduler_test.py

@@ -785,6 +785,16 @@ async def test_dynamic_timeout(self):
         scheduler = Scheduler(self.config, [DummyModel.remote(), DummyModel.remote()])
         await scheduler.start()
         tasks = []
+        tasks.extend(generate_tasks(0, timeout_num=4, repeat_times=1, timeout_seconds=1))
+        for task in tasks:
+            task.is_eval = True
+        scheduler.schedule(tasks, batch_id="0/eval")  # eval tasks will not count into dynamic timeout
+        statuses, exps = await scheduler.get_results(batch_id="0/eval")
+        self.assertEqual(len(statuses), 4)
+        self.assertEqual(len(exps), 0)
+        self.assertEqual(scheduler.total_running_time, 0)
+        self.assertEqual(scheduler.total_completed_tasks, 0)
+        tasks = []
         # generate 4 tasks that will run 1 second
         tasks.extend(generate_tasks(0, timeout_num=4, repeat_times=1, timeout_seconds=1))
         scheduler.schedule(tasks, batch_id=0)  # first step will not use dynamic timeout

tests/trainer/trainer_test.py

@@ -1005,10 +1005,16 @@ def test_trainer(self):
         self.config.algorithm.repeat_times = 4
         self.config.buffer.batch_size = 4
         self.config.buffer.total_steps = 2
-        self.config.buffer.explorer_input.taskset = get_unittest_dataset_config("gsm8k")
+        self.config.buffer.explorer_input.taskset = get_unittest_dataset_config("countdown", "train")
+        self.config.buffer.explorer_input.eval_tasksets = [
+            get_unittest_dataset_config("countdown", "test")
+        ]
+        self.config.buffer.eval_interval = 4  # only eval on start
         self.config.name = f"explore-over-rollout-{datetime.now().strftime('%Y%m%d%H%M%S')}"
         self.config.explorer.over_rollout.ratio = 0.5  # set over rollout rate to 50%, which means only wait for 2 (4 * 50%) tasks in each steps
         self.config.explorer.over_rollout.wait_after_min = 0
+        self.config.explorer.dynamic_timeout.enable = True
+        self.config.explorer.dynamic_timeout.ratio = 2
         self.config.algorithm.algorithm_type = "grpo"
         self.config.algorithm.advantage_fn = "grpo"
         self.config.algorithm.advantage_fn_args = {
@@ -1022,7 +1028,7 @@ def test_trainer(self):
         rollout_metrics = parser.metric_list("rollout")
         self.assertTrue(len(rollout_metrics) > 0)
         eval_metrics = parser.metric_list("eval")
-        self.assertTrue(len(eval_metrics) == 0)
+        self.assertTrue(len(eval_metrics) > 0)
         self.assertEqual(parser.metric_max_step(rollout_metrics[0]), 2)
         self.assertTrue(parser.metric_exist("experience_pipeline/experience_count"))
         experience_counts = parser.metric_values("experience_pipeline/experience_count")

trinity/explorer/scheduler.py

@@ -312,8 +312,9 @@ def task_done_callback(self, async_task: asyncio.Task):
             return
         else:
             status, exps, runner_id, run_time = async_task.result()
-            self.total_running_time += run_time
-            self.total_completed_tasks += 1
+            if not task.task.is_eval:  # only count running time for non-eval tasks
+                self.total_running_time += run_time
+                self.total_completed_tasks += 1
             task.results.append((status, exps))
             self.busy_runners.pop(runner_id)
             self.idle_runners.add(runner_id)