Fix stream without prompt format (#75)

* fix bug for stream generate of llamacpp

* fix steam without prompt format

* update log

Author: HaiHui886

llmserve/backend/llm/pipelines/llamacpp/llamacpp_pipeline.py

@@ -255,15 +255,11 @@ def streamGenerate(self, prompt: str, **generate_kwargs) -> Generator[str, None,
                     val = delta['content']
                 yield val
         else:
-            generate_kwargs.pop('max_tokens', None)
-            input_ids = self.tokenizer.encode(inputs[0])
-            # logger.info(f"model generate : {input_ids}")
             logger.info(f"generate_kwargs: {generate_kwargs}")
-            output = self.model.generate(tokens=input_ids, **generate_kwargs)
+            output = self.model(inputs[0], stream=True, **generate_kwargs)
             for token in output:
-                val = self.model.detokenize([token])
-                # logger.info(f'LlamaCppPipeline -> generate -> Yield -> "{val}" -> "{type(val)}"')
-                chunk = val.decode('utf-8')
+                # logger.info(f'LlamaCppPipeline -> generate -> Yield -> "{token}" -> "{type(token)}"')
+                chunk = token["choices"][0]["text"].replace("\u200b", "")
                 logger.info(f'LlamaCppPipeline -> generate -> Yield -> "{chunk}"')
                 yield chunk
 

models/text-generation--Qwen1.5-72B-Chat-GGUF.yaml

@@ -16,7 +16,7 @@ model_config:
   warmup: True
   model_task: text-generation
   model_id: Qwen/Qwen1.5-72B-Chat-GGUF
-  max_input_words: 512
+  max_input_words: 1024
   initialization:
     # s3_mirror_config:
       # bucket_uri: /data/models/Qwen1.5-72B-Chat-GGUF/
@@ -31,7 +31,7 @@ model_config:
     max_batch_size: 1
     batch_wait_timeout_s: 0
     generate_kwargs:
-      max_tokens: 512
+      max_tokens: 1024
     prompt_format: '[{{"role": "system", "content": "You are a helpful assistant."}},{{"role": "user", "content": "{instruction}"}}]'
     stopping_sequences: ["<|im_end|>"]
 scaling_config: