update

Signed-off-by: eric-higgins-ai <[email protected]>

Author: eric-higgins-ai

vllm/executor/ray_utils.py

@@ -16,6 +16,7 @@
 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
 from vllm.utils import get_ip
 from vllm.worker.worker_base import WorkerWrapperBase
+import vllm.envs as envs
 
 if TYPE_CHECKING:
     from vllm.v1.core.sched.output import SchedulerOutput
@@ -338,6 +339,7 @@ def initialize_ray_cluster(
     else:
         logger.info("No current placement group found. "
                     "Creating a new placement group.")
+        device_resource_request = envs.VLLM_RAY_PER_WORKER_GPUS
         num_devices_in_cluster = ray.cluster_resources().get(device_str, 0)
         # Log a warning message and delay resource allocation failure response.
         # Avoid immediate rejection to allow user-initiated placement group
@@ -349,7 +351,7 @@ def initialize_ray_cluster(
                 device_str)
         # Create a new placement group
         placement_group_specs: List[Dict[str, float]] = ([{
-            device_str: 1.0
+            device_str: device_resource_request
         } for _ in range(parallel_config.world_size)])
 
         # vLLM engine is also a worker to execute model with an accelerator,
@@ -358,11 +360,11 @@ def initialize_ray_cluster(
         current_ip = get_ip()
         current_node_id = ray.get_runtime_context().get_node_id()
         current_node_resource = available_resources_per_node()[current_node_id]
-        if current_node_resource.get(device_str, 0) < 1:
+        if current_node_resource.get(device_str, 0) < device_resource_request:
             raise ValueError(
                 f"Current node has no {device_str} available. "
                 f"{current_node_resource=}. vLLM engine cannot start without "
-                f"{device_str}. Make sure you have at least 1 {device_str} "
+                f"{device_str}. Make sure you have at least {device_resource_request} {device_str} "
                 f"available in a node {current_node_id=} {current_ip=}.")
         # This way, at least bundle is required to be created in a current
         # node.