[Serve] Fix OpenAIIngress scale-to-zero when all models have min_replicas=0

Signed-off-by: thjung123 <jeothen@gmail.com>

Author: thjung123

python/ray/llm/_internal/serve/core/ingress/ingress.py

@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import json
 import sys
 from contextlib import asynccontextmanager
@@ -97,6 +98,23 @@
     },
 }
 
+
+def _get_min_replicas_from_llm_config(config: LLMConfig) -> Optional[int]:
+    autoscaling_config = config.deployment_config.get("autoscaling_config")
+    if autoscaling_config is None:
+        return None
+    if isinstance(autoscaling_config, dict):
+        return autoscaling_config.get("min_replicas")
+    return getattr(autoscaling_config, "min_replicas", None)
+
+
+def _all_models_scale_to_zero(llm_configs: Optional[List[LLMConfig]]) -> bool:
+    """Check if all models are configured with min_replicas == 0."""
+    if not llm_configs:
+        return False
+    return all(_get_min_replicas_from_llm_config(config) == 0 for config in llm_configs)
+
+
 # These methods correspond to functions defined in the LLMEngine class in python/ray/llm/_internal/serve/deployments/llm/llm_engine.py
 class CallMethod(Enum):
     CHAT = "chat"
@@ -763,10 +781,17 @@ def get_deployment_options(
     ) -> Dict[str, Any]:
         """Get the deployment options for the ingress deployment.
 
+        If all models are configured with min_replicas=0 (scale-to-zero),
+        the ingress will also be configured with min_replicas=0 so that
+        the worker node/GPU instance can be fully released when idle.
+
         Args:
             llm_configs: The LLM configs to infer the number of ingress replicas from.
 
         Returns:
             A dictionary containing the deployment options for the ingress deployment.
         """
-        return DEFAULT_INGRESS_OPTIONS
+        options = copy.deepcopy(DEFAULT_INGRESS_OPTIONS)
+        if _all_models_scale_to_zero(llm_configs):
+            options.setdefault("autoscaling_config", {})["min_replicas"] = 0
+        return options

python/ray/llm/tests/serve/cpu/deployments/routers/test_builder_ingress.py

@@ -372,6 +372,113 @@ def test_user_target_ongoing_requests_respected(
         assert autoscaling_config.target_ongoing_requests == user_target
 
 
+class TestIngressScaleToZero:
+    """Tests for ingress scale-to-zero behavior when all models have min_replicas=0."""
+
+    def test_all_models_scale_to_zero(self, disable_placement_bundles):
+        """When all models have min_replicas=0, ingress should also have min_replicas=0."""
+        llm_cfg_dict_autoscaling = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_a"),
+            accelerator_type="L4",
+            deployment_config={
+                "autoscaling_config": {
+                    "min_replicas": 0,
+                    "max_replicas": 2,
+                }
+            },
+        )
+        llm_cfg_obj_autoscaling = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_b"),
+            accelerator_type="L4",
+            deployment_config={
+                "autoscaling_config": AutoscalingConfig(
+                    min_replicas=0,
+                    max_replicas=4,
+                )
+            },
+        )
+
+        app = build_openai_app(
+            LLMServingArgs(
+                llm_configs=[llm_cfg_dict_autoscaling, llm_cfg_obj_autoscaling],
+            )
+        )
+        autoscaling_config = app._bound_deployment._deployment_config.autoscaling_config
+        assert autoscaling_config.min_replicas == 0
+
+    def test_mixed_min_replicas_keeps_default(self, disable_placement_bundles):
+        """When some models have min_replicas>0, ingress should keep default min_replicas."""
+        llm_cfg_zero = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_a"),
+            accelerator_type="L4",
+            deployment_config={
+                "autoscaling_config": {
+                    "min_replicas": 0,
+                    "max_replicas": 2,
+                }
+            },
+        )
+        llm_cfg_nonzero = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_b"),
+            accelerator_type="L4",
+            deployment_config={
+                "autoscaling_config": AutoscalingConfig(
+                    min_replicas=1,
+                    max_replicas=4,
+                )
+            },
+        )
+
+        app = build_openai_app(
+            LLMServingArgs(
+                llm_configs=[llm_cfg_zero, llm_cfg_nonzero],
+            )
+        )
+        autoscaling_config = app._bound_deployment._deployment_config.autoscaling_config
+        # Default min_replicas from AutoscalingConfig is 1
+        assert autoscaling_config.min_replicas == 1
+
+    def test_no_autoscaling_config_keeps_default(self, disable_placement_bundles):
+        """When models don't have autoscaling_config, ingress should keep default."""
+        llm_cfg = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_a"),
+            accelerator_type="L4",
+        )
+
+        app = build_openai_app(
+            LLMServingArgs(llm_configs=[llm_cfg]),
+        )
+        autoscaling_config = app._bound_deployment._deployment_config.autoscaling_config
+        assert autoscaling_config.min_replicas == 1
+
+    def test_user_override_takes_precedence(self, disable_placement_bundles):
+        """User-specified ingress min_replicas should override scale-to-zero logic."""
+        llm_cfg = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_a"),
+            accelerator_type="L4",
+            deployment_config={
+                "autoscaling_config": {
+                    "min_replicas": 0,
+                    "max_replicas": 2,
+                }
+            },
+        )
+
+        app = build_openai_app(
+            LLMServingArgs(
+                llm_configs=[llm_cfg],
+                ingress_deployment_config={
+                    "autoscaling_config": {
+                        "min_replicas": 3,
+                        "max_replicas": 5,
+                    }
+                },
+            )
+        )
+        autoscaling_config = app._bound_deployment._deployment_config.autoscaling_config
+        assert autoscaling_config.min_replicas == 3
+
+
 def extract_applications_from_output(output: bytes) -> dict:
     """
     Extracts the 'applications' block from mixed output and returns it as a dict.