Fix/0412 (#690)

Author: tastelikefeet

docs/source/LLM/命令行参数.md

@@ -157,6 +157,7 @@
 dpo参数继承了sft参数, 除此之外增加了以下参数:
 
 - `--ref_model_type`: 对比模型的类型, 可以选择的`model_type`可以查看`MODEL_MAPPING.keys()`.
+- `--ref_model_id_or_path`: 对比模型的本地cache路径, 默认为`None`.
 - `--max_prompt_length`: 最大的提示长度, 该参数会传入DPOTrainer中, 使prompt长度不超过该值的设置, 默认值`1024`.
 - `--beta`: DPO logits的正则项，默认为0.1.
 - `--label_smoothing`: 是否使用DPO smoothing, 默认值为0，一般设置在0~0.5之间.
@@ -240,7 +241,7 @@ eval参数继承了infer参数，除此之外增加了以下参数：
 - `--eval_dataset`: 评测的官方数据集，默认值为`['ceval', 'gsm8k', 'arc']`, 此外支持`mmlu`和`bbh`两个数据集. 如果仅需要评测自定义数据集，可以将该参数设置为`no`.
 - `--eval_limit`: 每个评测集的子数据集的采样数量, 默认为`None`代表全量评测.
 - `--eval_few_shot`: 每个评测集的子数据集的few-shot个数, 默认为`None`代表使用数据集默认配置.
-- `--custom_eval_config`: 使用自定义数据集进行评测, 需要是一个本地存在的文件路径, 文件格式详见[自定义评测集](./LLM评测文档#自定义评测集).
+- `--custom_eval_config`: 使用自定义数据集进行评测, 需要是一个本地存在的文件路径, 文件格式详见[自定义评测集](./LLM评测文档.md#自定义评测集).
 
 ## app-ui 参数
 

docs/source_en/LLM/Command-line-parameters.md

@@ -157,6 +157,7 @@ The following parameters take effect when `sft_type` is set to `ia3`.
 dpo parameters inherit from sft parameters, with the following added parameters:
 
 - `--ref_model_type`: Type of reference model, available `model_type` options can be found in `MODEL_MAPPING.keys()`.
+- `--ref_model_id_or_path`: The local cache dir for reference model, default `None`.
 - `--max_prompt_length`: Maximum prompt length, this parameter is passed to DPOTrainer, setting prompt length to not exceed this value, default is `1024`.
 - `--beta`: Regularization term for DPO logits, default is 0.1.
 - `--label_smoothing`: Whether to use DPO smoothing, default is 0, generally set between 0~0.5.

swift/llm/dpo.py

@@ -58,9 +58,12 @@ def llm_dpo(args: DPOArguments) -> str:
         model_id_or_path=args.model_id_or_path,
         **kwargs)
     if args.ref_model_type is not None:
-        ref_model, _ = get_model_tokenizer(args.ref_model_type,
-                                           args.torch_dtype, model_kwargs,
-                                           **kwargs)
+        ref_model, _ = get_model_tokenizer(
+            args.ref_model_type,
+            args.torch_dtype,
+            model_kwargs,
+            model_id_or_path=args.ref_model_id_or_path,
+            **kwargs)
     else:
         ref_model = None
 

swift/llm/eval.py

@@ -8,8 +8,7 @@
 from modelscope import GenerationConfig
 
 from swift.utils import get_logger, get_main
-from . import (EvalArguments, inference, inference_vllm, merge_lora,
-               prepare_model_template)
+from . import EvalArguments, inference, merge_lora, prepare_model_template
 
 logger = get_logger()
 
@@ -39,6 +38,7 @@ def __init__(self, args: EvalArguments, model_name, config={}, **kwargs):
 
     def predict(self, prompt: str, **kwargs):
         if self.args.infer_backend == 'vllm':
+            from . import inference_vllm
             request_list = [{
                 'query': prompt,
                 'history': kwargs.get('history'),

swift/llm/utils/argument.py

@@ -1,14 +1,12 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import inspect
 import math
 import os
 from dataclasses import dataclass, field
-from typing import Dict, List, Literal, Optional, Set, Tuple, Union
+from typing import List, Literal, Optional, Set, Tuple, Union
 
 import json
 import numpy as np
 import torch
-import torch.distributed as dist
 import transformers
 from datasets import Dataset as HfDataset
 from datasets import concatenate_datasets
@@ -29,7 +27,7 @@
                       register_dataset)
 from .model import (MODEL_MAPPING, dtype_mapping, get_additional_saved_files,
                     get_default_lora_target_modules, get_default_template_type)
-from .template import TEMPLATE_MAPPING, TemplateType
+from .template import TEMPLATE_MAPPING
 from .utils import is_vllm_available
 
 logger = get_logger()
@@ -845,6 +843,8 @@ class DPOArguments(SftArguments):
         default=None,
         metadata={'help': f'model_type choices: {list(MODEL_MAPPING.keys())}'})
 
+    ref_model_id_or_path: Optional[str] = None
+
     max_prompt_length: int = 1024
     beta: float = 0.1
     label_smoothing: float = 0.0
@@ -1169,6 +1169,9 @@ def load_from_ckpt_dir(args: InferArguments) -> None:
             continue
         setattr(args, key, sft_args.get(key))
 
+    if args.model_id_or_path is None:
+        args.model_id_or_path = sft_args.get('model_id_or_path')
+
 
 def check_flash_attn(args: Union[SftArguments, InferArguments]) -> None:
     model_info = MODEL_MAPPING[args.model_type]

swift/llm/utils/dataset.py

@@ -549,7 +549,8 @@ def map_row(row):
         if response and response.startswith('Answer:'):
             response = response[len('Answer:') + 1:].strip()
         return {'query': row['query'], 'response': response}
-    return dataset.rename_columns({'instruction': 'query', 'output': 'response'})\
+
+    return dataset.rename_columns({'instruction': 'query', 'output': 'response'}) \
         .remove_columns(['input', 'file']).map(map_row).filter(lambda row: row['response'] is not None)
 
 
@@ -897,36 +898,56 @@ def process_hh_rlhf_cn(dataset):
 
     def reorganize_row(row):
         history = []
-        if isinstance(row['context'], str):
-            row['context'] = ast.literal_eval(row['context'])
-        if isinstance(row['chosen'], str):
-            row['chosen'] = ast.literal_eval(row['chosen'])
-        if isinstance(row['rejected'], str):
-            row['rejected'] = ast.literal_eval(row['rejected'])
-        for idx, h in enumerate(row['context']):
-            if idx % 2 == 0 and h['role'] != 'human':
-                return {'query': None}
-            if idx % 2 != 0 and h['role'] != 'assistant':
-                return {'query': None}
-            if idx % 2 == 0:
-                history.append([h['text'], None])
-            else:
-                history[-1][-1] = h['text']
-        if history[-1][-1] is not None:
-            return {'query': None}
-        query = history[-1][0]
-        history = history[:-1]
-        response = row['chosen']['text']
-        rejected_response = row['rejected']['text']
+        try:
+            if isinstance(row['context'], str):
+                row['context'] = ast.literal_eval(row['context'])
+            if isinstance(row['chosen'], str):
+                row['chosen'] = ast.literal_eval(row['chosen'])
+            if isinstance(row['rejected'], str):
+                row['rejected'] = ast.literal_eval(row['rejected'])
+            for idx, h in enumerate(row['context']):
+                if idx % 2 == 0 and h['role'] != 'human':
+                    raise ValueError()
+                if idx % 2 != 0 and h['role'] != 'assistant':
+                    raise ValueError()
+                if idx % 2 == 0:
+                    history.append([h['text'], None])
+                else:
+                    history[-1][-1] = h['text']
+            if history[-1][-1] is not None:
+                raise ValueError()
+            query = history[-1][0]
+            history = history[:-1]
+            response = row['chosen']['text']
+            rejected_response = row['rejected']['text']
+        except:  # noqa
+            return {
+                'query': '',
+                'response': '',
+                'rejected_response': '',
+                'history': [],
+            }
         return {
             'query': query,
             'response': response,
             'rejected_response': rejected_response,
             'history': history,
         }
 
-    return dataset.map(reorganize_row).filter(
-        lambda row: row['query'] is not None)
+    def row_can_be_parsed(row):
+        try:
+            if isinstance(row['context'], str):
+                row['context'] = ast.literal_eval(row['context'])
+            if isinstance(row['chosen'], str):
+                row['chosen'] = ast.literal_eval(row['chosen'])
+            if isinstance(row['rejected'], str):
+                row['rejected'] = ast.literal_eval(row['rejected'])
+            return True
+        except:  # noqa
+            return False
+
+    return dataset.filter(row_can_be_parsed).map(reorganize_row).filter(
+        lambda row: row['query'])
 
 
 register_dataset(