Allow queuing up to queued_requests_limit in one scheduler loop

Signed-off-by: Samuel Monson <[email protected]>

Author: sjmonson

src/guidellm/config.py

@@ -136,7 +136,7 @@ class Settings(BaseSettings):
         # use number of CPUs - 1, but at least 10
         default_factory=lambda: max((os.cpu_count() or 1) - 1, 10)
     )
-    max_add_requests_per_loop: int = 20
+    min_queued_requests: int = 20
     scheduler_start_delay: float = 5
 
     # Data settings

src/guidellm/scheduler/scheduler.py

@@ -140,7 +140,6 @@ async def run(
                 queues.requests,
                 times_iter,
                 run_info,
-                loop_limit=run_info.strategy.queued_requests_limit,
             )
             # Wait for the test to start
             await asyncio.sleep(time.time() - scheduling_strategy.start_time)
@@ -296,14 +295,14 @@ def _add_requests(
         requests_queue: Queue[WorkerProcessRequest[RequestT, ResponseT]],
         times_iter: Iterator[float],
         run_info: SchedulerRunInfo,
-        loop_limit: Optional[int] = None,
     ) -> Optional[Iterator[Any]]:
         if requests_iter is not None:
             try:
                 added_count = 0
 
                 while not requests_queue.full() and added_count < (
-                    loop_limit or settings.max_add_requests_per_loop
+                    run_info.strategy.queued_requests_limit
+                    or settings.min_queued_requests
                 ):
                     if run_info.created_requests >= run_info.end_number:
                         raise StopIteration