fix: switch from "nextTick" to "nextCycle" for the default batch dispatcher

for improved performance

Author: giladgd

src/evaluator/LlamaContext/LlamaContext.ts

@@ -74,7 +74,7 @@ export class LlamaContext {
         flashAttention = _model.defaultContextFlashAttention,
         threads,
         batching: {
-            dispatchSchedule: batchingDispatchSchedule = "nextTick",
+            dispatchSchedule: batchingDispatchSchedule = "nextCycle",
             itemPrioritizationStrategy: batchingItemsPrioritizationStrategy = "maximumParallelism"
         } = {},
         performanceTracking = false,
@@ -675,10 +675,19 @@ export class LlamaContext {
 
         if (this._queuedDecodeSequenceIds.size === this._totalSequences)
             dispatch();
-        if (dispatchSchedule === "nextTick")
-            setTimeout(dispatch, 0);
-        else
+        if (dispatchSchedule === "nextCycle") {
+            if (typeof setImmediate === "function")
+                setImmediate(dispatch);
+            else
+                setTimeout(dispatch, 0);
+        } else if (typeof dispatchSchedule === "function")
             dispatchSchedule(dispatch);
+        else {
+            if (typeof setImmediate === "function")
+                setImmediate(dispatch);
+            else
+                setTimeout(dispatch, 0);
+        }
     }
 
     /** @internal */

src/evaluator/LlamaContext/types.ts

@@ -215,12 +215,12 @@ export type LlamaContextSequenceRepeatPenalty = {
 export type BatchingOptions = {
     /**
      * The strategy used to dispatch items to be processed when there are items pending to be processed.
-     * - **`"nextTick"`** - dispatch the items on the next event loop tick.
+     * - **`"nextCycle"`** - dispatch the items on the next event loop cycle.
      * You can provide a custom function to define a custom dispatch schedule.
      *
-     * Defaults to `"nextTick"`.
+     * Defaults to `"nextCycle"`.
      */
-    dispatchSchedule?: "nextTick" | CustomBatchingDispatchSchedule,
+    dispatchSchedule?: "nextCycle" | CustomBatchingDispatchSchedule,
 
     /**
      * The strategy used to prioritize pending items to be processed.