make scale out policy consistent between deployments (#70)

Author: depenglee1707

llmserve/backend/server/run.py

@@ -78,8 +78,12 @@ def llm_server(args: Union[str, LLMApp, List[Union[LLMApp, str]]]):
             user_config=user_config,
             **deployment_config,
         ).bind()
-    # test = []
-    return RouterDeployment.bind(deployments, model_configs)  # pylint:disable=no-member
+
+    return RouterDeployment.options(
+        name=_reverse_prefix(model.model_config.model_id) + "-route",
+        max_concurrent_queries=max_concurrent_queries,
+        **deployment_config,
+    ).bind(deployments, model_configs)  # pylint:disable=no-member
 
 
 def llm_experimental(args: Union[str, LLMApp, List[Union[LLMApp, str]]]):

models/text-generation--facebook--opt-125m.yaml

@@ -1,14 +1,15 @@
 deployment_config:
+  max_concurrent_queries: 100
   autoscaling_config:
     min_replicas: 1
     initial_replicas: 1
-    max_replicas: 8
-    target_num_ongoing_requests_per_replica: 1.0
+    max_replicas: 3
+    target_num_ongoing_requests_per_replica: 10
     metrics_interval_s: 10.0
     look_back_period_s: 30.0
     smoothing_factor: 1.0
     downscale_delay_s: 300.0
-    upscale_delay_s: 90.0
+    upscale_delay_s: 60.0
   ray_actor_options:
     num_cpus: 0.1    # for a model deployment, we have 3 actor created, 1 and 2 will cost 0.1 cpu, and the model infrence will cost 6(see the setting in the end of the file)
 model_config:
@@ -48,4 +49,4 @@ model_config:
 scaling_config:
   num_workers: 1
   num_gpus_per_worker: 0
-  num_cpus_per_worker: 3   # for inference
+  num_cpus_per_worker: 2   # for inference