update code (#169)

Jintao-Huang · web-flow · commit c5ad6351e143 · 2023-11-23T10:42:01.000+08:00
diff --git a/README.md b/README.md
@@ -66,7 +66,7 @@ Users can check the [documentation of SWIFT](docs/source/GetStarted/快速使用
 
 
 ## ✨ LLM SFT Example
-The detailed usage documentation for fine-tuning LLM can be found [here](https://github.com/modelscope/swift/tree/main/examples/pytorch/llm).
+Users can refer to the [LLM fine-tuning documentation](https://github.com/modelscope/swift/tree/main/examples/pytorch/llm) for more detailed information.
 
 ### Features
 - Supported SFT Methods: [lora](https://arxiv.org/abs/2106.09685), [qlora](https://arxiv.org/abs/2305.14314), full(full parameter fine-tuning)
@@ -180,7 +180,7 @@ model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map'
 model = Swift.from_pretrained(model, model_dir, inference_mode=True)
 template = get_template(template_type, tokenizer)
 query = 'xxxxxx'
-response, history = inference(model, template, query, verbose=False)
+response, history = inference(model, template, query)
 print(f'response: {response}')
 print(f'history: {history}')
 ```
@@ -204,7 +204,7 @@ model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map'
 
 template = get_template(template_type, tokenizer)
 query = 'xxxxxx'
-response, history = inference(model, template, query, verbose=False)
+response, history = inference(model, template, query)
 print(f'response: {response}')
 print(f'history: {history}')
 ```
diff --git a/README_CN.md b/README_CN.md
@@ -64,7 +64,7 @@ SWIFT（Scalable lightWeight Infrastructure for Fine-Tuning）是一个可扩展
 
 
 ## ✨ 大模型微调的例子
-LLM微调的详细使用文档可以查看[这里](https://github.com/modelscope/swift/tree/main/examples/pytorch/llm).
+用户可以查看[LLM微调文档](https://github.com/modelscope/swift/tree/main/examples/pytorch/llm)来获得更详细的介绍.
 
 ### 特性
 - 支持的SFT方法: [lora](https://arxiv.org/abs/2106.09685), [qlora](https://arxiv.org/abs/2305.14314), 全参数微调
@@ -177,7 +177,7 @@ model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map'
 model = Swift.from_pretrained(model, model_dir, inference_mode=True)
 template = get_template(template_type, tokenizer)
 query = 'xxxxxx'
-response, history = inference(model, template, query, verbose=False)
+response, history = inference(model, template, query)
 print(f'response: {response}')
 print(f'history: {history}')
 ```
@@ -201,7 +201,7 @@ model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map'
 
 template = get_template(template_type, tokenizer)
 query = 'xxxxxx'
-response, history = inference(model, template, query, verbose=False)
+response, history = inference(model, template, query)
 print(f'response: {response}')
 print(f'history: {history}')
 ```
diff --git a/examples/pytorch/llm/README.md b/examples/pytorch/llm/README.md
@@ -143,7 +143,7 @@ model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map'
 model = Swift.from_pretrained(model, model_dir, inference_mode=True)
 template = get_template(template_type, tokenizer)
 query = 'xxxxxx'
-response, history = inference(model, template, query, verbose=False)
+response, history = inference(model, template, query)
 print(f'response: {response}')
 print(f'history: {history}')
 ```
@@ -167,7 +167,7 @@ model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map'
 
 template = get_template(template_type, tokenizer)
 query = 'xxxxxx'
-response, history = inference(model, template, query, verbose=False)
+response, history = inference(model, template, query)
 print(f'response: {response}')
 print(f'history: {history}')
 ```
diff --git a/examples/pytorch/llm/README_CN.md b/examples/pytorch/llm/README_CN.md
@@ -142,7 +142,7 @@ model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map'
 model = Swift.from_pretrained(model, model_dir, inference_mode=True)
 template = get_template(template_type, tokenizer)
 query = 'xxxxxx'
-response, history = inference(model, template, query, verbose=False)
+response, history = inference(model, template, query)
 print(f'response: {response}')
 print(f'history: {history}')
 ```
@@ -166,7 +166,7 @@ model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map'
 
 template = get_template(template_type, tokenizer)
 query = 'xxxxxx'
-response, history = inference(model, template, query, verbose=False)
+response, history = inference(model, template, query)
 print(f'response: {response}')
 print(f'history: {history}')
 ```
diff --git a/examples/pytorch/llm/scripts/tongyi_finance_14b_chat_int4/qlora/sft.sh b/examples/pytorch/llm/scripts/tongyi_finance_14b_chat_int4/qlora/sft.sh
@@ -11,6 +11,7 @@ python llm_sft.py \
     --dtype fp16 \
     --output_dir output \
     --custom_train_dataset_path xxx.jsonl \
+    --custom_val_dataset_path yyy.jsonl \
     --train_dataset_sample -1 \
     --num_train_epochs 1 \
     --max_length 4096 \
diff --git a/swift/cli/merge_lora.py b/swift/cli/merge_lora.py
@@ -1,6 +1,4 @@
-from swift.llm import InferArguments, merge_lora
-from swift.utils import parse_args
+from swift.llm.run import merge_lora_main
 
 if __name__ == '__main__':
-    args, remaining_argv = parse_args(InferArguments)
-    merge_lora(args, replace_if_exists=True)
+    merge_lora_main(replace_if_exists=True)
diff --git a/swift/llm/infer.py b/swift/llm/infer.py
@@ -18,7 +18,7 @@
 logger = get_logger()
 
 
-def merge_lora(args: InferArguments, replace_if_exists=False) -> None:
+def merge_lora(args: InferArguments, replace_if_exists=False) -> str:
     logger.info(f'replace_if_exists: {replace_if_exists}')
     assert args.ckpt_dir is not None
     assert args.sft_type == 'lora'
@@ -66,7 +66,7 @@ def merge_lora(args: InferArguments, replace_if_exists=False) -> None:
             res.pop('adapter_cfg', None)
             with open(new_configuration_path, 'w') as f:
                 json.dump(res, f, ensure_ascii=False, indent=4)
-        # sft_args
+        # sft_args.json
         sft_args_fname = 'sft_args.json'
         old_sft_args_path = os.path.join(old_ckpt_dir, sft_args_fname)
         new_sft_args_path = os.path.join(args.ckpt_dir, sft_args_fname)
@@ -80,7 +80,9 @@ def merge_lora(args: InferArguments, replace_if_exists=False) -> None:
     else:
         logger.info(
             f'The weight directory for the merged LoRA already exists in {args.ckpt_dir}, '
-            'skipping the saving process.')
+            'skipping the saving process. '
+            'you can pass `replace_if_exists=True` to overwrite it.')
+    return merged_lora_path
 
 
 def prepare_model_template(
@@ -152,7 +154,8 @@ def llm_infer(args: InferArguments) -> None:
     if args.eval_human:
         while True:
             query = input('<<< ')
-            _, history = inference(model, template, query, stream=args.stream)
+            _, history = inference(
+                model, template, query, stream=args.stream, verbose=True)
             item = history[0]
             if jsonl_path is not None:
                 save_result_to_jsonl(jsonl_path, item[0], item[1])
@@ -175,7 +178,8 @@ def llm_infer(args: InferArguments) -> None:
                 data.get('query'),
                 data.get('history'),
                 data.get('system'),
-                stream=args.stream)
+                stream=args.stream,
+                verbose=True)
             label = data.get('response')
             item = history[0]
             if jsonl_path is not None:
diff --git a/swift/llm/rome.py b/swift/llm/rome.py
@@ -75,7 +75,7 @@ def rome_infer(args: RomeArguments) -> None:
     if args.eval_human:
         while True:
             query = input('<<< ')
-            inference(model, template, query, stream=args.stream)
+            inference(model, template, query, stream=args.stream, verbose=True)
     else:
         _, val_dataset = get_dataset(args.dataset, args.dataset_test_ratio,
                                      args.dataset_seed)
@@ -88,7 +88,8 @@ def rome_infer(args: RomeArguments) -> None:
                 data.get('query'),
                 data.get('history'),
                 data.get('system'),
-                stream=args.stream)
+                stream=args.stream,
+                verbose=True)
             print()
             print(f"[LABELS]{data.get('response')}")
             print('-' * 80)
diff --git a/swift/llm/run.py b/swift/llm/run.py
@@ -1,8 +1,9 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 from swift.llm import (InferArguments, RomeArguments, SftArguments, get_main,
-                       llm_infer, llm_sft, llm_web_ui, rome_infer)
+                       llm_infer, llm_sft, llm_web_ui, merge_lora, rome_infer)
 
 sft_main = get_main(SftArguments, llm_sft)
 infer_main = get_main(InferArguments, llm_infer)
 rome_main = get_main(RomeArguments, rome_infer)
 web_ui_main = get_main(InferArguments, llm_web_ui)
+merge_lora_main = get_main(InferArguments, merge_lora)
diff --git a/swift/llm/sft.py b/swift/llm/sft.py
@@ -267,7 +267,7 @@ def llm_sft(args: SftArguments) -> str:
                     f,
                     ensure_ascii=False,
                     indent=2)
-    res = trainer.train(training_args.resume_from_checkpoint)
+    trainer.train(training_args.resume_from_checkpoint)
     logger.info(
         f'best_model_checkpoint: {trainer.state.best_model_checkpoint}')
 
@@ -283,6 +283,6 @@ def llm_sft(args: SftArguments) -> str:
     return {
         'best_model_checkpoint': trainer.state.best_model_checkpoint,
         'best_metric': trainer.state.best_metric,
-        'global_step': res.global_step,
+        'global_step': trainer.state.global_step,
         'log_history': trainer.state.log_history,
     }
diff --git a/swift/llm/utils/argument.py b/swift/llm/utils/argument.py
@@ -15,7 +15,8 @@
                          get_dist_setting, is_dist, is_master)
 from .dataset import (DATASET_MAPPING, DatasetName, get_custom_dataset,
                       register_dataset)
-from .model import MODEL_MAPPING, ModelType, dtype_mapping
+from .model import (MODEL_MAPPING, ModelType, dtype_mapping,
+                    get_default_template_type)
 from .template import TEMPLATE_MAPPING, TemplateType
 
 logger = get_logger()
@@ -195,7 +196,7 @@ def __post_init__(self) -> None:
             raise ValueError(f'sft_type: {self.sft_type}')
 
         if self.template_type == 'AUTO':
-            self.template_type = MODEL_MAPPING[self.model_type]['template']
+            self.template_type = get_default_template_type(self.model_type)
             logger.info(f'Setting template_type: {self.template_type}')
         if self.dataset is None:
             self.dataset = [DatasetName.blossom_math_zh]
@@ -322,7 +323,7 @@ def __post_init__(self) -> None:
 
         self.torch_dtype, _, _ = select_dtype(self)
         if self.template_type == 'AUTO':
-            self.template_type = MODEL_MAPPING[self.model_type]['template']
+            self.template_type = get_default_template_type(self.model_type)
             logger.info(f'Setting template_type: {self.template_type}')
         if self.dataset is None:
             self.dataset = [DatasetName.blossom_math_zh]
@@ -356,7 +357,7 @@ def __post_init__(self) -> None:
 
         self.torch_dtype, _, _ = select_dtype(self)
         if self.template_type == 'AUTO':
-            self.template_type = MODEL_MAPPING[self.model_type]['template']
+            self.template_type = get_default_template_type(self.model_type)
             logger.info(f'Setting template_type: {self.template_type}')
 
         if self.max_length == -1:
@@ -454,7 +455,8 @@ def set_model_type(args: Union[SftArguments, InferArguments]) -> None:
         args.model_type = model_mapping_reversed[model_id_or_path_lower]
 
     if args.model_type is None:
-        args.model_type = ModelType.qwen_7b_chat
+        raise ValueError(f'args.model_type: {args.model_type}, '
+                         f'args.model_id_or_path: {args.model_id_or_path}')
     if args.model_type not in MODEL_MAPPING:
         raise ValueError(f'model_type: {args.model_type} is not registered.')
     model_info = MODEL_MAPPING[args.model_type]
diff --git a/swift/llm/utils/utils.py b/swift/llm/utils/utils.py
@@ -372,8 +372,9 @@ def inference(model: PreTrainedModel,
               query: Optional[str] = None,
               history: Optional[History] = None,
               system: Optional[str] = None,
-              stream: bool = True,
-              verbose: bool = True,
+              *,
+              stream: bool = False,
+              verbose: bool = False,
               prompt_prefix: str = '[PROMPT]',
               output_prefix: str = '[OUTPUT]') -> Tuple[str, History]:
     if history is None:
@@ -386,15 +387,18 @@ def inference(model: PreTrainedModel,
     attention_mask = torch.ones_like(input_ids).to(device)
     model.eval()
     generation_config = getattr(model, 'generation_config', None)
-    if verbose:
-        print(
-            f'{prompt_prefix}{tokenizer.decode(input_ids[0], False)}{output_prefix}',
-            end='')
-    else:
+    if stream is True and verbose is False:
+        logger.warning(
+            'Please set verbose to True to support TextStreamer, or use `inference_stream.`'
+        )
         stream = False
     streamer = None
     if stream:
         streamer = TextStreamer(tokenizer, skip_prompt=True)
+    if verbose:
+        print(
+            f'{prompt_prefix}{tokenizer.decode(input_ids[0], False)}{output_prefix}',
+            end='')
     if generation_config.max_new_tokens is not None:
         generation_config.max_length = 20  # fix max_length, max_new_tokens warning
     generate_ids = model.generate(
@@ -403,7 +407,7 @@ def inference(model: PreTrainedModel,
         streamer=streamer,
         generation_config=generation_config)
     response = tokenizer.decode(generate_ids[0, len(input_ids[0]):], True)
-    if verbose and not streamer:
+    if verbose and stream is False:
         print(tokenizer.decode(generate_ids[0, len(input_ids[0]):], False))
     history.append((query, response))
     return response, history
diff --git a/tests/llm/test_run.py b/tests/llm/test_run.py
@@ -10,7 +10,7 @@
 import torch
 
 from swift.llm import DatasetName, InferArguments, ModelType, SftArguments
-from swift.llm.run import infer_main, sft_main
+from swift.llm.run import infer_main, merge_lora_main, sft_main
 
 
 class TestRun(unittest.TestCase):
@@ -48,8 +48,8 @@ def test_run_1(self):
             infer_args = InferArguments(
                 ckpt_dir=best_model_checkpoint,
                 stream=False,
-                show_dataset_sample=5,
-                merge_lora_and_save=True)
+                show_dataset_sample=5)
+            merge_lora_main(infer_args)
             result = infer_main(infer_args)
             print(result)
             torch.cuda.empty_cache()
diff --git a/tests/llm/test_template.py b/tests/llm/test_template.py
diff --git a/tests/utils/test_llm_utils.py b/tests/utils/test_llm_utils.py
diff --git a/tools/merge_lora_weights_to_model.py b/tools/merge_lora_weights_to_model.py
