Make vllm_async_service default handler for vllm and add default rolling_batch and async model options

Author: Suma Kasa

engines/python/src/main/java/ai/djl/python/engine/PyModel.java

@@ -187,6 +187,12 @@ public void load(Path modelPath, String prefix, Map<String, ?> options) throws I
                     recommendedEntryPoint = "djl_python.transformers_neuronx";
                 } else if ("trtllm".equals(features)) {
                     recommendedEntryPoint = "djl_python.tensorrt_llm";
+                } else if ("vllm".equals(features)) {
+                    recommendedEntryPoint = "djl_python.lmi_vllm.vllm_async_service";
+                    pyEnv.setAsyncMode(true);
+                    if (!properties.containsKey("rolling_batch")) {
+                        setProperty("rolling_batch", "disable");
+                    }
                 } else if (pyEnv.getInitParameters().containsKey("model_id")
                         || Files.exists(modelPath.resolve("config.json"))) {
                     recommendedEntryPoint = "djl_python.huggingface";

engines/python/src/test/java/ai/djl/python/engine/PyEngineTest.java

@@ -599,4 +599,43 @@ public void testRestartProcess() throws IOException, ModelException, Interrupted
             Assert.assertEquals(output.getCode(), 200);
         }
     }
+
+    @Test
+    public void testVllmFeaturesRollingBatch() throws IOException, ModelException {
+        System.setProperty("SERVING_FEATURES", "vllm");
+        try {
+            Criteria<Input, Output> criteria =
+                    Criteria.builder()
+                            .setTypes(Input.class, Output.class)
+                            .optModelPath(Paths.get("src/test/resources/echo"))
+                            .optEngine("Python")
+                            .build();
+            try (ZooModel<Input, Output> model = criteria.loadModel()) {
+                // Verify rolling_batch is set to disable when features=vllm
+                Assert.assertEquals(model.getProperty("rolling_batch"), "disable");
+            }
+        } finally {
+            System.clearProperty("SERVING_FEATURES");
+        }
+    }
+
+    @Test
+    public void testVllmFeaturesRollingBatchOverride() throws IOException, ModelException {
+        System.setProperty("SERVING_FEATURES", "vllm");
+        try {
+            Criteria<Input, Output> criteria =
+                    Criteria.builder()
+                            .setTypes(Input.class, Output.class)
+                            .optModelPath(Paths.get("src/test/resources/echo"))
+                            .optOption("rolling_batch", "vllm")
+                            .optEngine("Python")
+                            .build();
+            try (ZooModel<Input, Output> model = criteria.loadModel()) {
+                // Verify user override is respected
+                Assert.assertEquals(model.getProperty("rolling_batch"), "vllm");
+            }
+        } finally {
+            System.clearProperty("SERVING_FEATURES");
+        }
+    }
 }

tests/integration/llm/prepare.py

@@ -1701,9 +1701,6 @@ def build_vllm_async_model(model):
         )
     options = vllm_model_list[model]
     options["engine"] = "Python"
-    options["option.rolling_batch"] = "disable"
-    options["option.async_mode"] = "true"
-    options["option.entryPoint"] = "djl_python.lmi_vllm.vllm_async_service"
 
     adapter_ids = options.pop("adapter_ids", [])
     adapter_names = options.pop("adapter_names", [])
@@ -1780,7 +1777,6 @@ def build_vllm_model(model):
         )
     options = vllm_model_list[model]
     options["engine"] = "Python"
-    options["option.rolling_batch"] = "vllm"
 
     adapter_ids = options.pop("adapter_ids", [])
     adapter_names = options.pop("adapter_names", [])