fix batch_infer pad_token & florence (#2725)

Author: Jintao-Huang

swift/llm/infer/infer_engine/utils.py

@@ -153,6 +153,5 @@ def prepare_generation_config(model_generation_config: GenerationConfig, request
 
     if generation_config.eos_token_id is None:
         generation_config.eos_token_id = tokenizer.eos_token_id
-    if generation_config.pad_token_id is None:
-        generation_config.pad_token_id = tokenizer.pad_token_id
+    generation_config.pad_token_id = tokenizer.pad_token_id
     return generation_config

swift/llm/model/model/microsoft.py

@@ -3,6 +3,8 @@
 from types import MethodType
 from typing import Any, Dict
 
+from transformers import AutoConfig
+
 from swift.llm import TemplateType
 from swift.utils import get_env_args
 from ..constant import LLMModelType, MLLMModelType
@@ -55,9 +57,12 @@ def get_model_tokenizer_florence(model_dir: str,
                                  model_kwargs: Dict[str, Any],
                                  load_model: bool = True,
                                  **kwargs):
+    model_config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+    model_config.vision_config.model_type = 'davit'  # fix merge-lora
+    if model_kwargs['device_map'] == 'auto':
+        model_kwargs['device_map'] = 'cuda:0'
+    kwargs['model_config'] = model_config
     with ignore_check_imports():
-        if model_kwargs['device_map'] == 'auto':
-            model_kwargs['device_map'] = 'cuda:0'
         model, processor = get_model_tokenizer_multimodal(model_dir, model_info, model_kwargs, load_model, **kwargs)
 
     if model is not None:

swift/llm/template/base.py

@@ -607,10 +607,9 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
             for key, _slice in zip(['prompt', 'answer'],
                                    [slice(0, total_len - answer_len),
                                     slice(total_len - answer_len, total_len)]):
-                res_context_list, loss_scale_list = self._simplify_context_list(res_context_list[_slice],
-                                                                                loss_scale_list[_slice], inputs)
-                input_ids, labels, loss_scale, tokenizer_kwargs = self._encode_context_list(
-                    res_context_list, loss_scale_list)
+                context_list, loss_scale = self._simplify_context_list(res_context_list[_slice],
+                                                                       loss_scale_list[_slice], inputs)
+                input_ids, labels, loss_scale, tokenizer_kwargs = self._encode_context_list(context_list, loss_scale)
                 encoded[f'{key}_input_ids'] = input_ids
                 if key == 'answer':
                     encoded['labels'] = labels

swift/llm/template/template/microsoft.py

@@ -13,8 +13,8 @@
 
 
 class FlorenceTemplate(Template):
-    # loss_scale = 'last_round'
-    # skip_prompt = False
+    # If it's an encoder-decoder architecture, the default settings are
+    # loss_scale: 'last_round' and skip_prompt: False.
     is_encoder_decoder = True
 
     @staticmethod
@@ -51,28 +51,32 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
         labels = encoded['labels']
         if labels is not None:
             labels = [0] + labels
-        pixel_values = processor.image_processor(
-            images, return_tensors='pt')['pixel_values'].to(self.config.torch_dtype)
-        encoded = {
-            'input_ids': input_ids,
-            'labels': labels,
-            'pixel_values': pixel_values,
-        }
+        if images:
+            pixel_values = processor.image_processor(
+                images, return_tensors='pt')['pixel_values'].to(self.config.torch_dtype)
+            encoded['pixel_values'] = pixel_values
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
         return encoded
 
     def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
         inputs_embeds = model.get_input_embeddings()(inputs['input_ids'])
-        image_features = model._encode_image(inputs['pixel_values'])
-        inputs_embeds, _ = model._merge_input_ids_with_image_features(image_features, inputs_embeds)
+        pixel_values = inputs.get('pixel_values')
+        if pixel_values is not None:
+            image_features = model._encode_image(pixel_values)
+            inputs_embeds, inputs['attention_mask'] = model._merge_input_ids_with_image_features(
+                image_features, inputs_embeds)
         return {'inputs_embeds': inputs_embeds}
 
     def decode(self, generate_ids: List[int], **kwargs) -> Any:
         response = super().decode(generate_ids, **kwargs)
         template_inputs = kwargs.get('template_inputs')
         images = template_inputs.images
+        image_size = None
+        if images:
+            image_size = (images[0].width, images[0].height)
         return json.dumps(
-            self.processor.post_process_generation(
-                response, task=template_inputs.query, image_size=(images[0].width, images[0].height)))
+            self.processor.post_process_generation(response, task=template_inputs.query, image_size=image_size))
 
 
 register_template(

swift/trainers/rlhf_trainer/rlhf_mixin.py

@@ -85,7 +85,7 @@ def __init__(self,
             if self.ref_model is not None:
                 disable_dropout_in_model(self.ref_model)
 
-        self.is_encoder_decoder = args.is_encoder_decoder
+        self.is_encoder_decoder = kwargs['template'].is_encoder_decoder
         self.aux_loss_enabled = getattr(model.config, 'output_router_logits', False)
         self._peft_has_been_casted_to_bf16 = False
         self.generate_during_eval = getattr(args, 'generate_during_eval', False)

tests/test_align/test_template/test_vision.py

@@ -108,6 +108,8 @@ def test_llava_hf():
 
 def test_florence():
     pt_engine = PtEngine('AI-ModelScope/Florence-2-base-ft')
+    _infer_model(pt_engine, messages=[{'role': 'user', 'content': 'who are you?'}], images=[])
+
     _infer_model(
         pt_engine,
         messages=[{
@@ -265,7 +267,7 @@ def test_molmoe():
     # test_pixtral()
     # test_llama_vision()
     # test_llava_hf()
-    # test_florence()
+    test_florence()
     # test_glm_edge_v()
     #
     # test_phi3_vision()
@@ -276,4 +278,4 @@ def test_molmoe():
     # test_qvq()
     # test_mplug_owl2()
     # test_molmo()
-    test_molmoe()
+    # test_molmoe()