update stream & fix bugs (#4842)

Author: Jintao-Huang

docs/source/Instruction/命令行参数.md

@@ -110,7 +110,8 @@
 - top_p: top_p参数，默认为None。读取generation_config.json。
 - repetition_penalty: 重复惩罚项。默认为None，读取generation_config.json。
 - num_beams: beam search的并行保留数量，默认为1。
-- 🔥stream: 流式输出，默认为`False`。
+- 🔥stream: 流式输出，默认为`None`，即使用交互式界面时为True，数据集批量推理时为False。
+  - "ms-swift<3.6"stream默认值为False。
 - stop_words: 除了eos_token外额外的停止词，默认为`[]`。
   - 注意：eos_token会在输出respsone中被删除，额外停止词会在输出中保留。
 - logprobs: 是否输出logprobs，默认为False。

docs/source_en/Instruction/Command-line-parameters.md

@@ -112,7 +112,8 @@ Refer to the [generation_config](https://huggingface.co/docs/transformers/main_c
 - top_p: The top_p parameter, defaults to None. It is read from generation_config.json.
 - repetition_penalty: The repetition penalty. Defaults to None and is read from generation_config.json.
 - num_beams: The number of beams reserved for parallel beam search, default is 1.
-- 🔥stream: Stream output, default is `False`.
+- 🔥stream: Streaming output. Default is `None`, which means it is set to True when using the interactive interface and False during batch inference on datasets.
+  - For "ms-swift<3.6", the default value of stream is False.
 - stop_words: Additional stop words beyond eos_token, default is`[]`.
   - Note: eos_token will be removed in the output response, whereas additional stop words will be retained in the output.
 - logprobs: Whether to output logprobs, default is False.

swift/llm/argument/app_args.py

@@ -19,6 +19,7 @@ class AppArguments(WebUIArguments, DeployArguments):
 
     lang: Literal['en', 'zh'] = 'en'
     verbose: bool = False
+    stream: bool = True
 
     def _init_torch_dtype(self) -> None:
         if self.base_url:

swift/llm/argument/base_args/base_args.py

@@ -153,6 +153,7 @@ def __post_init__(self):
         self._init_custom_register()
         self._import_external_plugins()
         self._init_model_kwargs()
+        self._init_stream()
         # The Seq2SeqTrainingArguments has a property called world_size, which cannot be assigned a value.
         self.rank, self.local_rank, self.global_world_size, self.local_world_size = get_dist_setting()
         logger.info(f'rank: {self.rank}, local_rank: {self.local_rank}, '

swift/llm/argument/base_args/generation_args.py

@@ -32,11 +32,15 @@ class GenerationArguments:
     repetition_penalty: Optional[float] = None
     num_beams: int = 1
 
-    stream: bool = False
+    stream: Optional[bool] = None
     stop_words: List[str] = field(default_factory=list)
     logprobs: bool = False
     top_logprobs: Optional[int] = None
 
+    def _init_stream(self):
+        if self.stream is None:
+            self.stream = False
+
     def get_request_config(self):
         if getattr(self, 'task_type') != 'causal_lm':
             return

swift/llm/argument/deploy_args.py

@@ -4,6 +4,7 @@
 
 from swift.llm import safe_snapshot_download
 from swift.utils import find_free_port, get_logger
+from .base_args import BaseArguments
 from .infer_args import InferArguments
 
 logger = get_logger()
@@ -66,7 +67,7 @@ def _init_ckpt_dir(self, adapters=None):
         return super()._init_ckpt_dir(self.adapters + list(self.adapter_mapping.values()))
 
     def _init_stream(self):
-        pass
+        return BaseArguments._init_stream(self)
 
     def _init_eval_human(self):
         pass

swift/llm/argument/infer_args.py

@@ -180,7 +180,8 @@ def _init_result_path(self, folder_name: str) -> None:
 
     def _init_stream(self):
         self.eval_human = not (self.dataset and self.split_dataset_ratio > 0 or self.val_dataset)
-
+        if self.stream is None:
+            self.stream = self.eval_human
         if self.stream and self.num_beams != 1:
             self.stream = False
             logger.info('Setting args.stream: False')
@@ -199,7 +200,6 @@ def __post_init__(self) -> None:
         VllmArguments.__post_init__(self)
         self._init_result_path('infer_result')
         self._init_eval_human()
-        self._init_stream()
         self._init_ddp()
 
     def _init_eval_human(self):

swift/llm/infer/utils.py

@@ -4,6 +4,7 @@
 from dataclasses import dataclass, field
 from typing import List, Literal, Optional
 
+from swift.llm.utils import update_generation_config_eos_token
 from swift.plugin import extra_tuners
 from swift.tuners import Swift
 from swift.utils import get_logger
@@ -144,4 +145,5 @@ def prepare_model_template(args, **kwargs):
     model, processor = args.get_model_processor(**kwargs)
     model = prepare_adapter(args, model)
     template = args.get_template(processor)
+    update_generation_config_eos_token(model.generation_config, template)
     return model, template

swift/llm/template/base.py

@@ -501,6 +501,8 @@ def encode(self,
                     lengths += value
         if self.is_training:
             encoded['length'] = max(lengths)
+        else:
+            encoded.pop('length', None)
         if return_template_inputs:
             encoded['template_inputs'] = inputs
         if not self.remove_unused_columns:

swift/llm/utils.py

@@ -300,3 +300,21 @@ def get_ckpt_dir(model_dir: str, adapters_dir: Optional[List[str]]) -> str:
             ckpt_dir = model_dir
             break
     return ckpt_dir
+
+
+def update_generation_config_eos_token(generation_config, template):
+    stop_words = template.template_meta.stop_words
+    eos_token_id = generation_config.eos_token_id
+    if isinstance(eos_token_id, int):
+        eos_token_id = [eos_token_id]
+    modified = False
+    for stop_word in stop_words:
+        if stop_word is None:
+            continue
+        if isinstance(stop_word, str):
+            stop_word = template._tokenize(stop_word)
+        if isinstance(stop_word, (list, tuple)) and len(stop_word) == 1 and stop_word[0] not in eos_token_id:
+            eos_token_id.append(stop_word[0])
+            modified = True
+    if modified:
+        generation_config.eos_token_id = eos_token_id