add Qwen1.5-72B-GGUF yaml and fix load json input error (#71)

* add Qwen1.5-72B-GGUF yaml and fix load json input error

* add logs

* check prompt format

* change function name of llamacpp

* check prompt format

* update parameters

* update comments

* update

* update doc

* update log

* update

* add Qwen1.5-72B-GGUF

* remove echo

* add echo

* comment log

* remove echo

* fix stream

Author: HaiHui886

docs/common_issues.md

@@ -4,6 +4,16 @@
 
 Transformer version should be 4.33.3.
 
+## Use Llamacpp with GPU
+
+By default `llama-cpp-python` was installed without GPU support. 
+
+Refer repo [llama-cpp-python](https://github.com/abetlen/llama-cpp-python) to reinstall package `llama-cpp-python` for GPU support
+
+Links for GPU:
+- https://github.com/abetlen/llama-cpp-python/issues/509#issuecomment-1739098588
+- https://github.com/abetlen/llama-cpp-python/issues/627#issuecomment-1722495987
+
 ## Launch model by Ray Job API
 
 ```

llmserve/backend/llm/engines/generic.py

@@ -120,7 +120,7 @@ def init_model(
             logger.info("start to test with single prompt")
             logger.info(f"warmpup prompt is: {warmup_inputs}")
             resp = generate(
-                [Prompt(prompt=warmup_inputs, use_prompt_format=False)],
+                [Prompt(prompt=warmup_inputs, use_prompt_format=True)],
                 pipeline,
                 **generate_kwargs,
             )
@@ -287,7 +287,7 @@ def ping(self) -> bool:
     async def worker_stream_generate_texts(self, prompt: str, **kwargs) -> Generator[str, None, None]: # type: ignore
         logger.info(f"Call PredictionWorker.worker_stream_generate_texts with kwargs: {kwargs}")
         for s in self.generator.streamGenerate(prompt, **kwargs):
-            logger.info(f"PredictionWorker.worker_stream_generate_texts -> yield ->{s}")
+            # logger.info(f"PredictionWorker.worker_stream_generate_texts -> yield ->{s}")
             yield s
 
 class GenericEngine(LLMEngine):
@@ -375,7 +375,7 @@ def slice_prompts(worker_num: int, worker_index: int, prompts: list[str]):
             else:
                 return prompts[slice_size * worker_index: slice_size * worker_index + slice_size]
 
-        logger.info('LLM Predictor do async predict')
+        logger.info('LLM GenericEngine do async predict')
 
         async with lock:
             # prediction = (

llmserve/backend/llm/initializers/llamacpp.py

@@ -65,6 +65,7 @@ def __init__(
         self.model_init_kwargs = model_init_kwargs
 
     def _get_model_init_kwargs(self) -> Dict[str, Any]:
+        logger.info(f"model_init_kwargs: {self.model_init_kwargs}")
         return {
             # -1 means all layers are offloaded to GPU
             "n_gpu_layers": 0 if self.device.type == "cpu" else -1,
@@ -75,10 +76,10 @@ def _get_model_init_kwargs(self) -> Dict[str, Any]:
         }
 
     def load_model(self, model_id: str) -> "Llama":
-        logger.info(
-            f"LlamaCppInitializer downloading {model_id} : {self.model_filename}")
+        logger.info(f"LlamaCppInitializer downloading {model_id} : {self.model_filename}")
         model_path = hf_hub_download(model_id, self.model_filename)
         logger.info(f"LlamaCppInitializer Loading model {model_path}")
+        logger.info(f"model_init_kwargs: {self._get_model_init_kwargs()}")
         # Lazy import to avoid issues on CPU head node
         from llama_cpp import Llama
 

llmserve/backend/llm/pipelines/llamacpp/llamacpp_pipeline.py

@@ -11,6 +11,12 @@
 from ..utils import decode_stopping_sequences_where_needed, construct_prompts
 import json
 
+from typing import Generator
+from transformers import TextIteratorStreamer
+from threading import Thread
+from queue import Empty
+import asyncio
+
 if TYPE_CHECKING:
     from llama_cpp import Llama, LogitsProcessorList, StoppingCriteriaList
 
@@ -100,11 +106,10 @@ def _add_default_generate_kwargs(
         return generate_kwargs
 
     def __call__(self, inputs: List[str], **kwargs) -> List[Response]:
-        logger.info(inputs)
-        inputs = construct_prompts(
-            inputs, prompt_format=self.prompt_format)
-
-        logger.info(inputs)
+        logger.info(f"prompt_format: {self.prompt_format}")
+        logger.info(f"before construct_prompts: {inputs}")
+        inputs = construct_prompts(inputs, prompt_format=self.prompt_format)
+        logger.info(f"after construct_prompts: {inputs}")
 
         tokenized_inputs = self.tokenizer.encode(inputs)
         kwargs = self._add_default_generate_kwargs(
@@ -116,9 +121,10 @@ def __call__(self, inputs: List[str], **kwargs) -> List[Response]:
         chat_completion = False
         try:
             inputs_bak = inputs
-            inputs = [json.loads(prompt) for prompt in inputs]
+            inputs = [json.loads(prompt, strict=False) for prompt in inputs]
             chat_completion = True
-        except:
+        except Exception as ex:
+            logger.error(f"Exception apply_chat_template: {ex}")
             logger.info("Seems no chat template from user")
             inputs = inputs_bak
 
@@ -218,3 +224,49 @@ def from_initializer(
             device=device,
             **kwargs,
         )
+
+    def streamGenerate(self, prompt: str, **generate_kwargs) -> Generator[str, None, None]:
+        logger.info(f"stream prompt: {prompt}")
+        inputs = construct_prompts(prompt, prompt_format=self.prompt_format)
+        logger.info(f"stream inputs: {inputs}")
+        chat_completion = False
+        try:
+            inputs_bak = inputs
+            inputs = [json.loads(prompt, strict=False) for prompt in inputs]
+            chat_completion = True
+        except Exception as ex:
+            logger.error(f"Exception apply_chat_template: {ex}")
+            logger.info("Seems no chat template from user")
+            inputs = inputs_bak
+            
+        logger.info(f"stream generate_kwargs: {generate_kwargs}")
+        logger.info(f"model inputs: {inputs}")
+        
+        if chat_completion:
+            generate_kwargs.pop('stopping_sequences', None)
+            logger.info(f"chat generate_kwargs: {generate_kwargs}")
+            output = self.model.create_chat_completion(messages=inputs[0], stream=True, **generate_kwargs)
+            for chunk in output:
+                logger.info(f'LlamaCppPipeline -> create_chat_completion -> Yield -> "{chunk}" -> "{type(chunk)}"')
+                delta = chunk['choices'][0]['delta']
+                val = ''
+                if 'role' in delta:
+                    val = ''
+                elif 'content' in delta:
+                    val = delta['content']
+                yield val
+        else:
+            input_ids = self.model.tokenizer(inputs)
+            output = self.model.generate(tokens=input_ids, **generate_kwargs)
+            for token in output:
+                val = self.model.detokenize([token])
+                logger.info(f'LlamaCppPipeline -> generate -> Yield -> "{val}" -> "{type(val)}"')
+                yield val
+
+        # streaming sample for test
+        # start = 0
+        # while True:
+        #     val = prompt + str(start)
+        #     logger.info(f"LlamaCppPipeline.streamGenerate -> yield -> {val}")
+        #     yield val
+        #     start += 1

llmserve/backend/server/app.py

@@ -411,7 +411,10 @@ async def query(self, *args) -> Dict[str, Dict[str, Any]]:
         else:
             prompts = args[0]
         logger.info(f"ExperimentalDeployment query.prompts {prompts}")
-        results = await asyncio.gather(*[(self._model.generate_text.remote(Prompt(prompt=prompts, use_prompt_format=False)))])
+        use_prompt_format = False
+        if self._model_configuration.model_config.generation.prompt_format:
+            use_prompt_format = True
+        results = await asyncio.gather(*[(self._model.generate_text.remote(Prompt(prompt=prompts, use_prompt_format=use_prompt_format)))])
         logger.info(f"ExperimentalDeployment query.results {results}")
         results = results[0]
         return results.generated_text

llmserve/backend/server/config.py

@@ -132,7 +132,8 @@
     "OpenCSG/opencsg-starcoder-v0.1": "./models/text-generation--opencsg--opencsg-starcoder-15B-v0.1-pipeline.yaml",
     "opencsg/opencsg-deepseek-coder-1.3b-v0.1": "./models/text-generation--opencsg--opencsg-deepseek-coder-1.3b-v0.1.yaml",
     "OpenCSG/opencsg-deepseek-coder-1.3b-v0.1": "./models/text-generation--opencsg--opencsg-deepseek-coder-1.3b-v0.1.yaml",
-    "Qwen/Qwen1.5-72B-Chat": "./models/text-generation--Qwen--Qwen1.5-72B-Chat.yaml"
+    "Qwen/Qwen1.5-72B-Chat": "./models/text-generation--Qwen--Qwen1.5-72B-Chat.yaml",
+    "Qwen/Qwen1.5-72B-Chat-GGUF": "./models/text-generation--Qwen1.5-72B-Chat-GGUF.yaml"
 }
 
 SERVE_RUN_HOST = "0.0.0.0"

models/text-generation--Qwen1.5-72B-Chat-GGUF.yaml

@@ -0,0 +1,40 @@
+deployment_config:
+  autoscaling_config:
+    min_replicas: 1
+    initial_replicas: 1
+    max_replicas: 8
+    target_num_ongoing_requests_per_replica: 1.0
+    metrics_interval_s: 10.0
+    look_back_period_s: 30.0
+    smoothing_factor: 1.0
+    downscale_delay_s: 300.0
+    upscale_delay_s: 90.0
+  ray_actor_options:
+    num_cpus: 2    # for a model deployment, we have 3 actor created, 1 and 2 will cost 0.1 cpu, and the model inference will cost 6(see the setting in the end of the file)
+model_config:
+  # stream: True
+  warmup: True
+  model_task: text-generation
+  model_id: Qwen/Qwen1.5-72B-Chat-GGUF
+  max_input_words: 512
+  initialization:
+    # s3_mirror_config:
+      # bucket_uri: /data/models/Qwen1.5-72B-Chat-GGUF/
+    initializer:
+      type: LlamaCpp
+      model_filename: qwen1_5-72b-chat-q5_k_m.gguf
+      model_init_kwargs:
+        test: true
+        n_gpu_layers: -1
+    pipeline: llamacpp
+  generation:
+    max_batch_size: 1
+    batch_wait_timeout_s: 0
+    generate_kwargs:
+      max_tokens: 512
+    prompt_format: '[{{"role": "system", "content": "You are a helpful assistant."}},{{"role": "user", "content": "{instruction}"}}]'
+    stopping_sequences: ["<|im_end|>"]
+scaling_config:
+  num_workers: 1
+  num_gpus_per_worker: 6
+  num_cpus_per_worker: 8   # for inference