ppo compat transformers>=4.47.* (#2964)

Jintao-Huang · web-flow · commit f2d48a02c1ce · 2025-01-22T19:16:35.000+08:00
diff --git a/examples/deploy/client/llm/chat/swift_client.py b/examples/deploy/client/llm/chat/swift_client.py
@@ -1,5 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import asyncio
 import os
 from typing import List
 
diff --git a/examples/deploy/reward_model/client.py b/examples/deploy/reward_model/client.py
@@ -1,5 +1,4 @@
-from typing import List
-
+# Copyright (c) Alibaba, Inc. and its affiliates.
 from swift.llm import InferClient, InferRequest
 
 if __name__ == '__main__':
diff --git a/examples/train/rlhf/ppo.sh b/examples/train/rlhf/ppo.sh
@@ -1,4 +1,5 @@
 # Currently, it only supports the case where the model and reward_model use the same template/tokenizer.
+# Currently, multimodal model PPO is not supported.
 nproc_per_node=4
 
 CUDA_VISIBLE_DEVICES=0,1,2,3 \
diff --git a/scripts/utils/run_model_info.py b/scripts/utils/run_model_info.py
@@ -26,7 +26,6 @@ def get_model_info_table():
     for model_type in ModelType.get_model_name_list():
         model_meta = MODEL_MAPPING[model_type]
         template = model_meta.template
-        r = ''
         for group in model_meta.model_groups:
             for model in group.models:
                 ms_model_id = model.ms_model_id
diff --git a/swift/llm/argument/base_args/quant_args.py b/swift/llm/argument/base_args/quant_args.py
@@ -1,6 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import Literal, Optional
 
 import torch
diff --git a/swift/llm/argument/merge_args.py b/swift/llm/argument/merge_args.py
@@ -1,6 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 from dataclasses import dataclass
-from typing import Optional
 
 from swift.utils import get_logger
 
diff --git a/swift/llm/argument/train_args.py b/swift/llm/argument/train_args.py
@@ -8,7 +8,7 @@
 from transformers.utils.versions import require_version
 
 from swift.plugin import LOSS_MAPPING
-from swift.trainers import IntervalStrategy, TrainerFactory
+from swift.trainers import TrainerFactory
 from swift.utils import (add_version_to_work_dir, get_logger, get_pai_tensorboard_dir, is_liger_available,
                          is_local_master, is_mp, is_pai_training_job, use_torchacc)
 from .base_args import BaseArguments, to_abspath
diff --git a/swift/llm/dataset/preprocessor/core.py b/swift/llm/dataset/preprocessor/core.py
@@ -173,7 +173,7 @@ def batched_preprocess(self, batched_row: Dict[str, Any], *, strict: bool,
                 elif self.traceback_limit is not None and self._traceback_counter < self.traceback_limit:
                     import traceback
                     logger.info(traceback.format_exc())
-                    logger.error('👆👆👆There are errors in the dataset, the data will be deleted')
+                    logger.warning('👆👆👆There are errors in the dataset, the data will be deleted')
                     self._traceback_counter += 1
                 row = []
             new_rows += row
diff --git a/swift/llm/dataset/utils.py b/swift/llm/dataset/utils.py
@@ -184,8 +184,8 @@ def __getitem__(self, idx: int) -> Dict[str, Any]:
                 if self.traceback_limit is not None and self._traceback_counter < self.traceback_limit:
                     import traceback
                     logger.info(traceback.format_exc())
-                    logger.error('👆👆👆There are errors in the template.encode, '
-                                 'and another piece of data will be randomly selected.')
+                    logger.warning('👆👆👆There are errors in the template.encode, '
+                                   'and another piece of data will be randomly selected.')
                     self._traceback_counter += 1
 
     def __len__(self) -> int:
diff --git a/swift/llm/export/merge_lora.py b/swift/llm/export/merge_lora.py
@@ -1,7 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import os
-import shutil
-import tempfile
 
 from swift.llm import ExportArguments, prepare_model_template, save_checkpoint
 from swift.tuners import Swift
diff --git a/swift/llm/infer/deploy.py b/swift/llm/infer/deploy.py
@@ -5,7 +5,6 @@
 import time
 from contextlib import contextmanager
 from dataclasses import asdict
-from functools import partial
 from http import HTTPStatus
 from threading import Thread
 from typing import List, Optional, Union
diff --git a/swift/llm/infer/infer.py b/swift/llm/infer/infer.py
@@ -1,7 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 from contextlib import nullcontext
 from itertools import chain
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Union
 
 import numpy as np
 import torch.distributed as dist
diff --git a/swift/llm/model/model/deepseek.py b/swift/llm/model/model/deepseek.py
@@ -55,9 +55,6 @@ def get_model_tokenizer_deepseek_moe(model_dir: str,
         # fix dtype bug
         mlp_cls = model.model.layers[1].mlp.__class__
 
-        def _dtype_hook(module, input, output):
-            return output.to(input[0].dtype)
-
         for module in model.modules():
             if isinstance(module, mlp_cls):
                 patch_output_to_input_device(module)
diff --git a/swift/llm/model/model/valley.py b/swift/llm/model/model/valley.py
@@ -7,8 +7,7 @@
 from swift.llm import TemplateType
 from ..constant import MLLMModelType
 from ..model_arch import ModelArch
-from ..register import (Model, ModelGroup, ModelMeta, get_model_tokenizer_multimodal,
-                        get_model_tokenizer_with_flash_attn, register_model)
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
 from ..utils import ModelInfo, git_clone_github, safe_snapshot_download
 
 
@@ -25,7 +24,6 @@ def get_model_tokenizer_valley(model_dir: str,
     sys.path.append(os.path.join(local_repo_path))
 
     if llm_model_type == 'valley':
-        from torch.nn import CrossEntropyLoss
         from transformers.modeling_outputs import CausalLMOutputWithPast
         from valley_eagle.model.language_model.valley_qwen2 import ValleyQwen2ForCausalLM, ValleyConfig
         model_config = ValleyConfig.from_pretrained(model_dir)
diff --git a/swift/llm/template/base.py b/swift/llm/template/base.py
@@ -354,7 +354,6 @@ def _concat_context_list(
             if isinstance(context, str):
                 if '{{RESPONSE}}' == context:
                     assert response is not None
-                    context = response
                     res_context_list.append(response)
                     res_context_type.append(ContextType.RESPONSE)
                     continue
diff --git a/swift/llm/template/template/llava.py b/swift/llm/template/template/llava.py
@@ -3,15 +3,13 @@
 from typing import Any, Dict, List, Literal, Optional
 
 import torch
-import transformers
-from packaging import version
 
 from ..base import Template
 from ..constant import MLLMTemplateType
 from ..register import TemplateMeta, register_template
 from ..template_inputs import StdTemplateInputs
 from ..utils import Context, Prompt, findall
-from ..vision_utils import load_batch, load_video_llava
+from ..vision_utils import load_video_llava
 from .llama import Llama3TemplateMeta
 from .qwen import QwenTemplateMeta
 from .utils import ChatmlTemplateMeta
diff --git a/swift/llm/template/vision_utils.py b/swift/llm/template/vision_utils.py
@@ -53,7 +53,7 @@ def _dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbn
 
     # calculate the existing image aspect ratio
     target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1)
-                        if i * j <= max_num and i * j >= min_num)
+                        if min_num <= i * j <= max_num)
     target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
 
     # find the closest aspect ratio to the target
diff --git a/swift/llm/train/sft.py b/swift/llm/train/sft.py
@@ -5,7 +5,7 @@
 
 from datasets import Dataset as HfDataset
 
-from swift.plugin import extra_callbacks, get_loss_func, get_metric, optimizers_map
+from swift.plugin import extra_callbacks, get_loss_func, get_metric
 from swift.trainers import IntervalStrategy, TrainerFactory
 from swift.utils import (append_to_jsonl, get_logger, get_model_parameter_info, is_master, plot_images, stat_array,
                          use_torchacc)
diff --git a/swift/trainers/rlhf_trainer/ppo_trainer.py b/swift/trainers/rlhf_trainer/ppo_trainer.py
@@ -1,6 +1,8 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 from contextlib import contextmanager
 
+import transformers
+from packaging import version
 from torch.utils.data import DataLoader
 from transformers import PreTrainedModel
 from trl import PPOv2Trainer as HFPPOv2Trainer
@@ -40,8 +42,15 @@ def __init__(self, model: PreTrainedModel, ref_model: PreTrainedModel, *_args, *
             ppo_trainer_init(
                 self, config=kwargs['args'], tokenizer=self.tokenizer, policy=model, ref_policy=ref_model, **new_kwargs)
         unwrap_model = self.accelerator.unwrap_model(self.model)
-        patch_getattr(unwrap_model, 'policy')
+        patch_getattr(unwrap_model.__class__, 'policy')
 
     def train(self, *args, **kwargs):
         # remove args that are not needed for the HFPPOTrainer
         super().train()
+
+    def _save_checkpoint(self, *args, **kwargs):
+        if version.parse(transformers.__version__) >= version.parse('4.47'):
+            metrics = kwargs.pop('metrics', None)
+            trial = kwargs.get('trial')
+            self._determine_best_metric(metrics=metrics, trial=trial)
+        return super()._save_checkpoint(*args, **kwargs)
diff --git a/swift/trainers/rlhf_trainer/reward_trainer.py b/swift/trainers/rlhf_trainer/reward_trainer.py
@@ -66,7 +66,7 @@ def visualize_samples(self, num_print_samples: int):
             table['rejected_text'].extend(gather_object(rejected_text))
             table['logits'].extend(
                 gather_object([[round(inner_item, 4) for inner_item in item] for item in logits.tolist()]))
-            if num_print_samples >= 0 and len(table['chosen_text']) >= num_print_samples:
+            if 0 <= num_print_samples <= len(table['chosen_text']):
                 break
         df = pd.DataFrame(table)
         if self.accelerator.process_index == 0:
diff --git a/swift/trainers/rlhf_trainer/rlhf_mixin.py b/swift/trainers/rlhf_trainer/rlhf_mixin.py
@@ -39,8 +39,9 @@ def load_state_dict(self, *args, **kwargs):
     def parameters(self, *args, **kwargs):
         return self._model.parameters(*args, **kwargs)
 
+    @staticmethod
     @contextmanager
-    def _save_load_context(cls, trainer):
+    def _save_load_context(trainer):
         # fix zero3 & save/load model
         deepspeed_model = trainer.deepspeed
         _new_model = deepspeed_model._model
diff --git a/swift/tuners/scetuning/scetuning.py b/swift/tuners/scetuning/scetuning.py
@@ -1,13 +1,11 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import inspect
 import re
 import types
 from dataclasses import dataclass, field
 from typing import List, Optional, Union
 
 import torch
 from torch import nn
-from transformers.activations import ACT2CLS
 
 from swift import get_logger
 from swift.tuners.utils import ActivationMixin, SwiftAdapter, SwiftConfig, SwiftOutput
diff --git a/swift/tuners/scetuning/scetuning_components.py b/swift/tuners/scetuning/scetuning_components.py
@@ -3,8 +3,6 @@
 
 import torch
 import torch.nn as nn
-import torch.nn.functional as F
-from einops import rearrange
 
 from swift.utils.logger import get_logger
 
diff --git a/swift/tuners/side.py b/swift/tuners/side.py
@@ -6,7 +6,7 @@
 from dataclasses import dataclass, field
 from functools import partial
 from itertools import repeat
-from typing import List, Union
+from typing import Union
 
 import torch
 from torch import nn
diff --git a/swift/tuners/utils.py b/swift/tuners/utils.py
@@ -6,7 +6,6 @@
 import shutil
 import tempfile
 import threading
-import uuid
 from dataclasses import asdict, dataclass, field
 from types import FunctionType
 from typing import Dict, Optional, Union
diff --git a/swift/ui/app.py b/swift/ui/app.py
@@ -1,7 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import os
-from copy import copy
-from dataclasses import fields
 from functools import partial
 from typing import List, Union
 
diff --git a/swift/ui/llm_infer/runtime.py b/swift/ui/llm_infer/runtime.py
@@ -7,7 +7,6 @@
 from typing import Dict, List, Tuple, Type
 
 import gradio as gr
-import json
 import psutil
 from packaging import version
 
diff --git a/tests/run.py b/tests/run.py
@@ -468,7 +468,7 @@ class TimeCostTextTestResult(TextTestResult):
 
     def __init__(self, stream, descriptions, verbosity):
         self.successes = []
-        return super(TimeCostTextTestResult,
+        super(TimeCostTextTestResult,
                      self).__init__(stream, descriptions, verbosity)
 
     def startTest(self, test):
diff --git a/tests/train/test_cls.py b/tests/train/test_cls.py
@@ -43,11 +43,7 @@ def test_mllm():
     from swift.llm import TrainArguments, sft_main, infer_main, InferArguments
     result = sft_main(
         TrainArguments(
-            model='Qwen/Qwen2-VL-7B-Instruct',
-            train_type='lora',
-            num_labels=2,
-            dataset=['DAMO_NLP/jd:cls#2000'],
-            **kwargs))
+            model='OpenGVLab/InternVL2-1B', train_type='lora', num_labels=2, dataset=['DAMO_NLP/jd:cls#500'], **kwargs))
     last_model_checkpoint = result['last_model_checkpoint']
     infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True))
 
diff --git a/tests/train/test_ppo.py b/tests/train/test_ppo.py
@@ -48,7 +48,23 @@ def test_ppo2():
     infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True, merge_lora=True))
 
 
+def test_ppo_vl():
+    # PPO currently does not support VL, and the image has not been uploaded.
+    os.environ['MAX_PIXELS'] = '1003520'
+    from swift.llm import rlhf_main, RLHFArguments, infer_main, InferArguments
+    result = rlhf_main(
+        RLHFArguments(
+            rlhf_type='ppo',
+            model='Qwen/Qwen2-VL-2B-Instruct',
+            reward_model='Qwen/Qwen2-VL-7B-Instruct',
+            dataset=['modelscope/coco_2014_caption:validation#100'],
+            **kwargs))
+    last_model_checkpoint = result['last_model_checkpoint']
+    infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True, merge_lora=True))
+
+
 if __name__ == '__main__':
     # test_rm()
     # test_ppo()
-    test_ppo2()
+    # test_ppo2()
+    test_ppo_vl()

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`# Copyright (c) Alibaba, Inc. and its affiliates.`
`2`		`-import asyncio`
`3`	`2`	`import os`
`4`	`3`	`from typing import List`
`5`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`# Currently, it only supports the case where the model and reward_model use the same template/tokenizer.`
	`2`	`+# Currently, multimodal model PPO is not supported.`
`2`	`3`	`nproc_per_node=4`
`3`	`4`
`4`	`5`	`CUDA_VISIBLE_DEVICES=0,1,2,3 \`