update

Author: tchaton

src/litserve/loops/base.py

@@ -259,11 +259,9 @@ async def _wrapper():
                     except Exception as e:
                         logger.exception("An error occurred in the loop: %s", e)
 
-                    if schedule_task.done() and not lit_api.has_active_requests():
+                    if not lit_api.has_active_requests() and schedule_task.done():
                         self.on_schedule_task_done(schedule_task)
 
-                    await asyncio.sleep(0)
-
             event_loop.run_until_complete(_wrapper())
         else:
             while True:

src/litserve/loops/continuous_batching_loop.py

@@ -140,7 +140,7 @@ async def prefill(
 
         while True:
             if self.no_pending_requests and lit_api.has_active_requests():
-                await asyncio.sleep(0.001)
+                await asyncio.sleep(0)
                 return pending_requests
 
             request = await asyncio.to_thread(self.get_request, request_queue, timeout=1, block=True)

tests/unit/test_lit_server.py

@@ -752,6 +752,7 @@ def encode_response(self, output):
         return {"output": output}
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Test is only for Unix")
 @pytest.mark.asyncio
 async def test_worker_restart_and_server_shutdown():
     api = FailingLitAPI()