Update ray to 2.9.3 (#56)

* Update ray to 2.9.3

* update prompt

Author: HaiHui886

README.md

@@ -65,13 +65,13 @@ pip install '.[vllm]' -i https://pypi.tuna.tsinghua.edu.cn/simple/
 Pip install Ray:
 
 ```
-pip install -U "ray[serve-grpc]==2.8.0"
+pip install -U "ray[serve-grpc]==2.9.3"
 ```
 
 Option to use another pip source for faster transfer if needed.
 
 ```
-pip install -U "ray[serve-grpc]==2.8.0" -i https://pypi.tuna.tsinghua.edu.cn/simple/
+pip install -U "ray[serve-grpc]==2.9.3" -i https://pypi.tuna.tsinghua.edu.cn/simple/
 ```
 
 > **Note:** ChatGLM2-6b requires transformers<=4.33.3, while the latest vllm requires transformers>=4.36.0.
@@ -82,7 +82,7 @@ Start cluster then:
 ray start --head --port=6379 --dashboard-host=0.0.0.0 --dashboard-port=8265
 ```
 
-See reference [here](https://docs.ray.io/en/releases-2.8.0/ray-overview/installation.html).
+See reference [here](https://docs.ray.io/en/releases-2.9.3/ray-overview/installation.html).
 
 #### Quick start
 

deploy/ray/Dockerfile-base

@@ -5,5 +5,5 @@ RUN sudo apt-get update && sudo apt-get install -y libaio-dev git-lfs awscli &&
 RUN conda update -n base -c defaults conda -y
 RUN conda install python=3.10 -y
 RUN pip install --upgrade pip
-RUN pip install -U "ray[serve-grpc]==2.8.0"
+RUN pip install -U "ray[serve-grpc]==2.9.3"
 RUN pip install -i https://download.pytorch.org/whl/cu118 torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2

llmserve/backend/server/app.py

@@ -194,6 +194,7 @@ async def generate_text(self, prompt: Prompt):
                 # priority=QueuePriority.GENERATE_TEXT,
                 # start_timestamp=start_timestamp,
             )
+            logger.info(f"generated text: {text}")
             # return text[0]
             return text
 
@@ -351,13 +352,9 @@ async def predict(self, model: str, prompt: Union[Prompt, List[Prompt]]) -> Unio
                 logger.info(f"set modelID: {item}")
         logger.info(f"search model key {modelID}")
         if isinstance(prompt, Prompt):
-            results = await asyncio.gather(
-                *(await asyncio.gather(*[self._models[modelID].generate_text.remote(prompt)]))
-            )
+            results = await asyncio.gather(*[self._models[modelID].generate_text.remote(prompt)])
         elif isinstance(prompt, list):
-            results = await asyncio.gather(
-                *(await asyncio.gather(*[self._models[modelID].batch_generate_text.remote(prompt)]))
-            )
+            results = await asyncio.gather(*[self._models[modelID].batch_generate_text.remote(prompt)])
         else:
             raise Exception("Invaid prompt format.")
         logger.info(f"{results}")
@@ -418,9 +415,7 @@ async def query(self, *args) -> Dict[str, Dict[str, Any]]:
         else:
             prompts = args[0]
         logger.info(f"ExperimentalDeployment query.prompts {prompts}")
-        results = await asyncio.gather(
-            *(await asyncio.gather(*[self._model.generate_text.remote(Prompt(prompt=prompts, use_prompt_format=False))]))
-        )
+        results = await asyncio.gather(*[(self._model.generate_text.remote(Prompt(prompt=prompts, use_prompt_format=False)))])
         logger.info(f"ExperimentalDeployment query.results {results}")
         results = results[0]
         return results.generated_text

models/text-generation--Qwen--Qwen1.5-72B-Chat.yaml

@@ -33,7 +33,7 @@ model_config:
     max_batch_size: 1
     batch_wait_timeout_s: 0
     generate_kwargs:
-      bos_token_id: 151643,
+      bos_token_id: 151643
       # pad_token_id: 151643,
       # eos_token_id: [151645, 151643],
       do_sample: false
@@ -42,7 +42,7 @@ model_config:
       temperature: 0.7
       top_p: 0.8
       top_k: 20
-    prompt_format: "'role': 'user', 'content': {instruction}"
+    prompt_format: '[{{"role": "system", "content": "You are a helpful assistant."}},{{"role": "user", "content": "{instruction}"}}]'
     # stopping_sequences: ["### Response:", "### End"]
 scaling_config:
   num_workers: 1

models/text-generation--gpt2.yaml

@@ -1,6 +1,6 @@
 deployment_config:
   autoscaling_config:
-    min_replicas: 0
+    min_replicas: 1
     initial_replicas: 1
     max_replicas: 8
     target_num_ongoing_requests_per_replica: 1.0