enable chat template for huggingface transformer (#54)

Author: depenglee1707

llmserve/backend/llm/pipelines/default_pipeline.py

@@ -6,6 +6,7 @@
 
 from llmserve.backend.logger import get_logger
 from llmserve.backend.server.models import Response
+import json
 
 from ._base import BasePipeline
 from .processors import StopOnTokens
@@ -54,6 +55,16 @@ def preprocess(self, prompts: List[str], **generate_kwargs):
         if self.tokenizer.pad_token is None:
             self.tokenizer.pad_token = self.tokenizer.eos_token
 
+        try:
+            prompt_text_bak = prompt_text
+            prompt_text = [json.loads(prompt) for prompt in prompt_text]
+            prompt_text = [self.tokenizer.apply_chat_template(prompt_obj, tokenize=False, add_generation_prompt=True) for prompt_obj in prompt_text]
+        except:
+            logger.info("Seems no chat template from user or the model donot has a 'chat template'")
+            prompt_text = prompt_text_bak
+
+        logger.info(f"Call model.generate with input: {prompt_text}")
+
         inputs = self.tokenizer(
             prompt_text, return_tensors="pt", add_special_tokens = generate_kwargs.get("add_special_tokens", True), padding=True
         ).to(self.model.device if hasattr(self.model, 'device') else self.device)

llmserve/backend/llm/pipelines/default_transformers_pipeline.py

@@ -2,14 +2,15 @@
 
 import torch
 import time
+import json
 from transformers import Pipeline as TransformersPipeline
 from transformers import PreTrainedModel, PreTrainedTokenizer, pipeline
 
 from llmserve.backend.logger import get_logger
 from llmserve.backend.server.models import Prompt, Response
 
 from ._base import BasePipeline
-from .utils import construct_prompts_experimental, truncate_to_first_stop_token
+from .utils import construct_prompts
 from llmserve.backend.server.utils import render_gradio_params
 from .default_pipeline import DefaultPipeline
 
@@ -135,12 +136,20 @@ def preprocess(self, prompts: List[str], **generate_kwargs):
         st = time.monotonic()
         inputs = None
         logger.info(f"input from pipeline: ****** {prompts}")
-        prompt_text = construct_prompts_experimental(
+        prompt_text = construct_prompts(
             prompts, prompt_format=self.prompt_format)
-        instruction_text = construct_prompts_experimental(prompts, prompt_format="")
+        instruction_text = construct_prompts(prompts, prompt_format="")
         logger.info(f"input from pipeline: ****** {prompt_text}")   
 
         if isinstance(self.pipeline, transformers.pipelines.text_generation.TextGenerationPipeline):
+            try:
+                prompt_text_bak = prompt_text
+                prompt_text = [json.loads(prompt) for prompt in prompt_text]
+                prompt_text = [self.tokenizer.apply_chat_template(prompt_obj, tokenize=False, add_generation_prompt=True) for prompt_obj in prompt_text]
+            except:
+                logger.info("Seems no chat template from user or the model donot has a 'chat template'")
+                prompt_text = prompt_text_bak
+
             inputs = self.tokenizer(
                 prompt_text, return_tensors="pt", add_special_tokens = generate_kwargs.get("add_special_tokens", True), padding=True
             )
@@ -224,7 +233,7 @@ def postprocess(self, model_outputs, **postprocess_kwargs) -> List[Response]:
                 output).input_ids)
             num_input_tokens = len(self.tokenizer(inputs[index]))
             response = Response(
-                generated_text=output,
+                generated_text=output[len(inputs[index]):],
                 num_generated_tokens=num_generated_tokens,
                 num_input_tokens=num_input_tokens,
             )

llmserve/backend/llm/pipelines/utils.py

@@ -69,25 +69,6 @@ def construct_prompts(
         prompts = [prompts]
     return [_construct_prompt(prompt, prompt_format) for prompt in prompts]
 
-
-def construct_prompts_experimental(
-    prompts: Union[str, Prompt, List[str], List[Prompt], Tuple[str]],
-    prompt_format: str,
-) -> List[str]:
-    """Construct prompts from a prompt string or list of prompts."""
-    if not isinstance(prompts, list):
-        prompts = [prompts]
-
-    params = []
-    for prompt in prompts:
-        if isinstance(prompt, Prompt) and isinstance(prompt.prompt, Tuple):
-            params += [_construct_prompt(prompt, prompt_format)
-                       for prompt in prompt.prompt]
-        else:
-            params.append(_construct_prompt(prompt, prompt_format))
-    return params
-
-
 def tokenize_stopping_sequences_where_needed(
     tokenizer: PreTrainedTokenizer,
     stopping_sequences: List[Union[str, int, List[int]]],

llmserve/backend/llm/predictor.py

@@ -113,7 +113,7 @@ def init_model(
             logger.info("start to test with single prompt")
             logger.info(f"warmpup prompt is: {warmup_inputs}")
             resp = generate(
-                [warmup_inputs],
+                [Prompt(prompt=warmup_inputs, use_prompt_format=False)],
                 pipeline,
                 **generate_kwargs,
             )

llmserve/backend/llm/utils.py

@@ -279,14 +279,6 @@ async def init_torch_dist_process_group_async(
         node_id = node_and_gpu_ids[rank][0]
         local_rank = node_to_workers[node_id].index(rank)
         local_world_size = len(node_to_workers[node_id])
-        logger.info("++++++++++++++")
-        logger.info(rank)
-        logger.info(world_size)
-        logger.info(local_rank)
-        logger.info(local_world_size)
-        logger.info(master_addr)
-        logger.info(master_port)
-        logger.info(list(node_to_gpu_ids[node_id]))
         setup_futures.append(
             worker.execute.remote(
                 _init_torch_distributed,

llmserve/backend/server/utils.py

@@ -175,7 +175,7 @@ def render_gradio_params(hg_task: str) -> Dict[str, Any]:
         pipeline_info = {
             "inputs": gr.components.Textbox(label="Input"),
             "outputs": gr.components.Textbox(label="Output"),
-            "preprocess": lambda x: {"text_inputs": [(text + "\n") for text in x]},
+            "preprocess": lambda x: {"text_inputs": [text for text in x]},
             # "postprocess": lambda r: r[0]["generated_text"],
             "postprocess": lambda r: [text[0]['generated_text'] for text in r],
             "warmup": "Write a short story."

models/text-generation--facebook--blenderbot-400M-distill.yaml

@@ -0,0 +1,48 @@
+deployment_config:
+  autoscaling_config:
+    min_replicas: 1
+    initial_replicas: 1
+    max_replicas: 8
+    target_num_ongoing_requests_per_replica: 1.0
+    metrics_interval_s: 10.0
+    look_back_period_s: 30.0
+    smoothing_factor: 1.0
+    downscale_delay_s: 300.0
+    upscale_delay_s: 90.0
+  ray_actor_options:
+    num_cpus: 0.1    # for a model deployment, we have 3 actor created, 1 and 2 will cost 0.1 cpu, and the model infrence will cost 6(see the setting in the end of the file)
+model_config:
+  warmup: True
+  model_task: text-generation
+  model_id: facebook/blenderbot-400M-distill
+  max_input_words: 48
+  initialization:
+    # s3_mirror_config:
+      # endpoint_url: http://39.107.108.170:9000 # Optinal for custom S3 storage endpoint url 
+      # bucket_uri: s3://opt-125m/facemodel/  # Must include hash file with commit id in repo
+      # bucket_uri: /tmp/hub/opt-125m/ # Local path of model with hash file
+      # git_uri: https://portal.opencsg.com/models/opt-125m.git # git address for git clone
+    initializer:
+      type: SingleDevice
+      dtype: float32
+      from_pretrained_kwargs:
+        use_cache: true
+        trust_remote_code: true
+    pipeline: default
+  generation:
+    max_batch_size: 1
+    batch_wait_timeout_s: 0
+    generate_kwargs:
+      do_sample: true
+      max_new_tokens: 24
+      min_new_tokens: 16
+      temperature: 0.7
+      repetition_penalty: 1.1
+      top_p: 0.8
+      top_k: 50
+    prompt_format: '[{{"role": "system", "content": "You are a friendly chatbot who always responds in the style of a pirate"}},{{"role": "user", "content": "{instruction}"}}]'
+    #stopping_sequences: ["### Response:", "### End"]
+scaling_config:
+  num_workers: 1
+  num_gpus_per_worker: 0
+  num_cpus_per_worker: 3   # for inference

setup.py

@@ -52,7 +52,7 @@
             "deepspeed==0.14.0",
             "torchmetrics==1.2.1",
             "llama_cpp_python==0.2.20",
-            "transformers==4.33.3",
+            "transformers==4.39.1",
         ],
         "vllm": [
             "vllm==0.2.7",