Qwen 2.5 VL

torchtune/models/qwen2_5/_model_builders.py

@@ -372,9 +372,9 @@ def qwen2_5_tokenizer(
         Qwen2_5Tokenizer: Instantiation of the Qwen2.5 tokenizer
     """
     special_tokens = (
-        parse_hf_tokenizer_json(special_tokens_path)
-        if special_tokens_path is not None
-        else None
+        QWEN2_5_SPECIAL_TOKENS
+        if special_tokens_path is None
+        else parse_hf_tokenizer_json(special_tokens_path)
     )
 
     if prompt_template is not None:

torchtune/modules/attention.py

@@ -4,6 +4,7 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import inspect
 import logging
 from typing import Optional
 
@@ -15,6 +16,36 @@
 logger = logging.getLogger(__name__)
 
 
+def _call_pos_embedding_safely(
+    pos_embedding: nn.Module,
+    x: torch.Tensor,
+    input_pos: Optional[torch.Tensor] = None,
+    window_index: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Call positional embedding with only the parameters it accepts.
+
+    Args:
+        pos_embedding (nn.Module): The positional embedding module
+        x (torch.Tensor): Input tensor
+        input_pos (Optional[torch.Tensor]): Optional input position tensor
+        window_index (Optional[torch.Tensor]): Optional window index tensor
+
+    Returns:
+        Output tensor from positional embedding
+    """
+    sig = inspect.signature(pos_embedding.forward)
+    kwargs = {}
+
+    # Only add parameters that the method accepts
+    if "input_pos" in sig.parameters:
+        kwargs["input_pos"] = input_pos
+    if "window_index" in sig.parameters:
+        kwargs["window_index"] = window_index
+
+    return pos_embedding(x, **kwargs)
+
+
 class MultiHeadAttention(nn.Module):
     """Multi-headed attention layer with support for grouped query
     attention (GQA) introduced in https://arxiv.org/abs/2305.13245v1.
@@ -242,7 +273,9 @@ def forward(
 
         # Apply positional embeddings
         if self.pos_embeddings is not None:
-            q = self.pos_embeddings(q, input_pos=input_pos, window_index=window_index)
+            q = _call_pos_embedding_safely(
+                self.pos_embeddings, q, input_pos, window_index
+            )
 
         # [b, n_h, s_x, h_d]
         q = q.transpose(1, 2)
@@ -270,8 +303,8 @@ def forward(
             k = k.view(b, s_y, -1, self.head_dim)
             v = v.view(b, s_y, -1, self.head_dim)
             if self.pos_embeddings is not None:
-                k = self.pos_embeddings(
-                    k, input_pos=input_pos, window_index=window_index
+                k = _call_pos_embedding_safely(
+                    self.pos_embeddings, k, input_pos, window_index
                 )
 
             # k,v shape: [b, n_kv, s_y, h_d]