compat trl 0.15 (#5905)

Author: Jintao-Huang

swift/llm/model/register.py

@@ -91,7 +91,7 @@ def get_matched_model_group(self, model_name: str) -> Optional[ModelGroup]:
                 for key in ['ms_model_id', 'hf_model_id', 'model_path']:
                     value = getattr(model, key)
 
-                    if isinstance(value, str) and model_name == value.rsplit('/', 1)[-1]:
+                    if isinstance(value, str) and model_name == value.rsplit('/', 1)[-1].lower():
                         return model_group
 
     def check_requires(self, model_info=None):
@@ -435,7 +435,7 @@ def get_all_models() -> List[str]:
 
 
 def get_matched_model_meta(model_id_or_path: str) -> Optional[ModelMeta]:
-    model_name = get_model_name(model_id_or_path)
+    model_name = get_model_name(model_id_or_path).lower()
     for model_type, model_meta in MODEL_MAPPING.items():
         model_group = ModelMeta.get_matched_model_group(model_meta, model_name)
         if model_group is not None:

swift/megatron/utils/convert.py

@@ -241,8 +241,7 @@ def convert_hf2mcore(args: ExportArguments) -> None:
 
 def convert_mcore2hf(args: ExportArguments) -> None:
     from swift.megatron import prepare_mcore_model, adapter_state_dict_context
-    hf_model, template = prepare_model_template(
-        args, load_model=args.to_hf, patch_offload=not args.test_convert_precision)
+    _, template = prepare_model_template(args, load_model=False)
     processor = template.processor
 
     megatron_model_meta = get_megatron_model_meta(args.model_type)
@@ -284,6 +283,7 @@ def convert_mcore2hf(args: ExportArguments) -> None:
         mg_model = peft_model.merge_and_unload()
     logger.info('Megatron model created successfully.')
     if args.to_hf:
+        hf_model = prepare_model_template(args, patch_offload=not args.test_convert_precision)[0]
         megatron_model_meta.convert_mcore2hf(hf_model, mg_model)
         if args.test_convert_precision:
             test_convert_precision(hf_model, mg_model, template, args.test_convert_dtype)

swift/trainers/rlhf_arguments.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, field
-from typing import List
+from typing import List, Optional
 
 from trl import CPOConfig as HfCPOConfig
 from trl import DPOConfig as HfDPOConfig
@@ -15,7 +15,7 @@
 
 @dataclass
 class DPOConfig(SwiftArgumentsMixin, HfDPOConfig):
-    pass
+    ld_alpha: Optional[float] = None  # compat trl==0.15
 
 
 @dataclass

swift/trainers/rlhf_trainer/dpo_trainer.py

@@ -7,6 +7,7 @@
 from accelerate.utils import gather_object
 from peft import PeftModel
 from transformers import PreTrainedModel
+from transformers.utils.versions import require_version
 from trl import DPOTrainer as HFDPOTrainer
 from trl.trainer.dpo_config import DPOConfig
 from trl.trainer.utils import RunningMoments, selective_log_softmax
@@ -70,6 +71,8 @@ def __init__(self,
 
         if 'bco_pair' in loss_types:
             self.running = RunningMoments(self.accelerator)
+        if self.args.ld_alpha is not None:
+            require_version('trl>=0.18', '`ld_alpha` requires that "trl>=0.18".')
         if self.template.packing:
             self.accelerator.gather_for_metrics = new_gather_function