rename loss_subbatch_seqlen to loss_subbatch_sequence_length (#2581)

Author: cheng221

paddleformers/nn/criterion/dpo_loss.py

@@ -83,7 +83,7 @@ def dpo_logps(
     #   bsz,seq_len,hidden_size or seq_len,hidden_size
     seq_len = labels.shape[1] if labels.ndim == 2 else labels.shape[0]
 
-    if self.use_fused_head_and_loss_fn and self.use_subbatch and seq_len > self.loss_subbatch_seqlen:
+    if self.use_fused_head_and_loss_fn and self.use_subbatch and seq_len > self.loss_subbatch_sequence_length:
         per_token_logps = -fused_head_and_loss_fn(
             hidden_states,
             weight,
@@ -95,7 +95,7 @@ def dpo_logps(
             self.config.tensor_parallel_degree,
             self.config.tensor_parallel_output,
             False,  # fused_linear
-            self.loss_subbatch_seqlen,
+            self.loss_subbatch_sequence_length,
             return_token_loss=True,
             ignore_index=0,
         )
@@ -120,12 +120,12 @@ def dpo_logps(
             logits = logits.unsqueeze(0)
         elif logits.dim() == 3 and labels.dim() == 1:
             labels = labels.unsqueeze(0)
-        if self.use_subbatch and seq_len > self.loss_subbatch_seqlen:
+        if self.use_subbatch and seq_len > self.loss_subbatch_sequence_length:
             sb_loss_func = subbatch(
                 self.loss_func,
                 [0, 1],
                 [1, 1],
-                self.loss_subbatch_seqlen,
+                self.loss_subbatch_sequence_length,
                 1,
             )
 

paddleformers/nn/criterion/interface.py

@@ -45,14 +45,14 @@ def __init__(self, config, return_tuple=True, ignore_eos_token=False, use_infohu
         self.kto_config = copy.deepcopy(config.get("kto_config", None))
         self.ignored_index = getattr(config, "ignored_index", -100)
         self.use_filtered_label_loss = config.get("use_filtered_label_loss", False)
-        self.loss_subbatch_seqlen = config.get("loss_subbatch_seqlen", -1)
-        self.use_subbatch = self.loss_subbatch_seqlen > 0
+        self.loss_subbatch_sequence_length = config.get("loss_subbatch_sequence_length", -1)
+        self.use_subbatch = self.loss_subbatch_sequence_length > 0
         self.sequence_parallel = config.get("sequence_parallel", False)
         self.tensor_parallel = config.tensor_parallel_degree > 1
         self.use_fused_head_and_loss_fn = config.get("use_fused_head_and_loss_fn", False)
         self.enable_parallel_cross_entropy = config.tensor_parallel_degree > 1 and config.tensor_parallel_output
         logger.info(
-            f"loss_subbatch_seqlen: {self.loss_subbatch_seqlen} , use_fused_head_and_loss_fn: {self.use_fused_head_and_loss_fn}, use_filtered_label_loss: {self.use_filtered_label_loss}"
+            f"loss_subbatch_sequence_length: {self.loss_subbatch_sequence_length} , use_fused_head_and_loss_fn: {self.use_fused_head_and_loss_fn}, use_filtered_label_loss: {self.use_filtered_label_loss}"
         )
 
         self.return_tuple = return_tuple

paddleformers/nn/criterion/kto_loss.py

@@ -96,7 +96,7 @@ def kto_logps(
 
     # bsz,seq_len,hidden_size or seq_len,hidden_size
     seq_len = labels.shape[1] if labels.ndim == 2 else labels.shape[0]
-    if self.use_fused_head_and_loss_fn and self.use_subbatch and seq_len > self.loss_subbatch_seqlen:
+    if self.use_fused_head_and_loss_fn and self.use_subbatch and seq_len > self.loss_subbatch_sequence_length:
         per_token_logps = -fused_head_and_loss_fn(
             hidden_states,
             weight,
@@ -108,7 +108,7 @@ def kto_logps(
             self.config.tensor_parallel_degree,
             self.config.tensor_parallel_output,
             self.config.fused_linear,
-            self.loss_subbatch_seqlen,
+            self.loss_subbatch_sequence_length,
             return_token_loss=True,
             ignore_index=self.ignored_index,
         )
@@ -133,12 +133,12 @@ def kto_logps(
         elif logits.dim() == 3 and labels.dim() == 1:
             labels = labels.unsqueeze(0)
 
-        if self.use_subbatch and seq_len > self.loss_subbatch_seqlen:
+        if self.use_subbatch and seq_len > self.loss_subbatch_sequence_length:
             sb_loss_func = subbatch(
                 self.loss_func,
                 [0, 1],
                 [1, 1],
-                self.loss_subbatch_seqlen,
+                self.loss_subbatch_sequence_length,
                 1,
             )
             per_token_logps = sb_loss_func(logits, labels.unsqueeze(-1))

paddleformers/nn/criterion/sft_loss.py

@@ -86,7 +86,7 @@ def sft_loss_forward(
     masked_lm_labels = labels
     # bsz,seq_len,hidden_size or seq_len,hidden_size
     seq_len = masked_lm_labels.shape[1] if masked_lm_labels.ndim == 2 else masked_lm_labels.shape[0]
-    if self.use_fused_head_and_loss_fn and self.use_subbatch and seq_len > self.loss_subbatch_seqlen:
+    if self.use_fused_head_and_loss_fn and self.use_subbatch and seq_len > self.loss_subbatch_sequence_length:
         masked_lm_loss = fused_head_and_loss_fn(
             hidden_states,
             lm_head_weight,
@@ -98,7 +98,7 @@ def sft_loss_forward(
             self.config.tensor_parallel_degree,
             self.config.tensor_parallel_output,
             False,
-            self.loss_subbatch_seqlen,
+            self.loss_subbatch_sequence_length,
             return_token_loss=True,
             ignore_index=self.ignored_index,
         )
@@ -131,12 +131,12 @@ def sft_loss_forward(
 
         # logits: bsz seq_len
         # labels: bsz seq_len vocab_size
-        if self.use_subbatch and seq_len > self.loss_subbatch_seqlen:
+        if self.use_subbatch and seq_len > self.loss_subbatch_sequence_length:
             sb_loss_func = subbatch(
                 self.loss_func,
                 [0, 1],
                 [1, 1],
-                self.loss_subbatch_seqlen,
+                self.loss_subbatch_sequence_length,
                 1,
             )
             masked_lm_loss = sb_loss_func(logits, labels.unsqueeze(-1))

paddleformers/transformers/configuration_utils.py

@@ -287,10 +287,10 @@ class LlmMetaConfig:
         ("use_fused_head_loss_fn", bool, False, "Whether to use fused head and loss function."),
         ("use_filtered_label_loss", bool, False, "Whether to use filtered label loss."),
         (
-            "loss_subbatch_seqlen",
+            "loss_subbatch_sequence_length",
             int,
             -1,
-            "Sequence length larger than loss_subbatch_seqlen will be divided into multiple subbatches during loss computation (-1 means disable subbatch).",
+            "Sequence length larger than loss_subbatch_sequence_length will be divided into multiple subbatches during loss computation (-1 means disable subbatch).",
         ),
     ]
 

tests/nn/test_criterion.py

@@ -37,7 +37,7 @@ def test_forward_default_sft(self):
 
     def test_forward_non_fuse_subbatch_sft(self):
         config = copy.deepcopy(self.config)
-        config.loss_subbatch_seqlen = 2
+        config.loss_subbatch_sequence_length = 2
         config.use_fused_head_and_loss_fn = False
         layer = CriterionLayer(config=config)
         layer(self.logits, self.labels)