fix ernie4.5 moe bug (#2689)

Author: cheng221

paddleformers/nn/moe/moe_allgather_layer.py

@@ -714,9 +714,9 @@ def forward_experts(self, *dispatched_input):
                 else:
                     input_shape = [
                         1,
-                        true_experts[iexpert].down_proj.lora_A.shape[1],
+                        true_experts[iexpert].down_proj.lora_B.shape[1],
                     ]
-                    input_dtype = true_experts[iexpert].down_proj.lora_A.dtype
+                    input_dtype = true_experts[iexpert].down_proj.lora_B.dtype
 
                 chunk = paddle.zeros(
                     input_shape,

paddleformers/transformers/configuration_utils.py

@@ -284,8 +284,14 @@ class LlmMetaConfig:
     ]
 
     loss_attributes = [
-        ("use_fused_head_loss_fn", bool, False, "Whether to use fused head and loss function."),
+        ("use_fused_head_and_loss_fn", bool, False, "Whether to use fused head and loss function."),
         ("use_filtered_label_loss", bool, False, "Whether to use filtered label loss."),
+        (
+            "use_sparse_head_and_loss_fn",
+            bool,
+            False,
+            "Maintained for compatibility, recommend using use_filtered_label_loss instead. (Legacy params)",
+        ),
         (
             "loss_subbatch_sequence_length",
             int,

paddleformers/transformers/model_utils.py

@@ -397,7 +397,7 @@ def _is_need_transpose(key):
 
     def _transpose_hf_weight(key, weight):
         if _is_need_transpose(key):
-            return weight.transpose([-1, -2])
+            return np.ascontiguousarray(weight.transpose([-1, -2]))
         return weight
 
     part_state_dict = {}