LLaVA OV: fix unpadding precision (huggingface#34779)

* fix

* propagate

* type check

Author: zucchini-nlp

src/transformers/models/llava_next/processing_llava_next.py

@@ -163,7 +163,9 @@ def __call__(
                 for sample in text:
                     while self.image_token in sample:
                         image_size = next(image_sizes)
-                        orig_height, orig_width = image_size
+                        if not isinstance(image_size, (list, tuple)):
+                            # cast to list to avoid numerical precision errors when calculating unpadding
+                            orig_height, orig_width = image_size.tolist()
                         num_image_tokens = self._get_number_of_features(orig_height, orig_width, height, width)
                         if self.vision_feature_select_strategy == "default":
                             num_image_tokens -= self.num_additional_image_tokens

src/transformers/models/llava_next_video/processing_llava_next_video.py

@@ -190,7 +190,9 @@ def __call__(
                 for sample in text:
                     while self.image_token in sample:
                         image_size = next(image_sizes)
-                        orig_height, orig_width = image_size
+                        if not isinstance(image_size, (list, tuple)):
+                            # cast to list to avoid numerical precision errors when calculating unpadding
+                            orig_height, orig_width = image_size.tolist()
                         num_image_tokens = self._get_number_of_features(orig_height, orig_width, height, width)
                         if self.vision_feature_select_strategy == "default":
                             num_image_tokens -= self.num_additional_image_tokens

src/transformers/models/llava_onevision/processing_llava_onevision.py

@@ -188,7 +188,10 @@ def _expand_image_tokens(
         for sample in text:
             while special_token in sample:
                 image_size_list = next(image_sizes)
-                orig_height, orig_width = image_size_list[0] if num_frames != 1 else image_size_list
+                original_size = image_size_list[0] if num_frames != 1 else image_size_list
+                if not isinstance(original_size, (list, tuple)):
+                    # cast to list to avoid numerical precision errors when calculating unpadding
+                    orig_height, orig_width = original_size.tolist()
                 num_image_tokens = self._get_number_of_features(orig_height, orig_width, height, width)
                 if self.vision_feature_select_strategy == "default":
                     num_image_tokens -= 1