cherry-pick #10545

Author: pkuzyc

paddlenlp/trainer/training_args.py

@@ -1405,7 +1405,7 @@ def is_context_parallel_supported():
                     else:
                         order = ["dp", "sharding", "pp", "mp"]
                 if self.use_expert_parallel:
-                    if self.moe_sharding_parallel_degree > 1 and self.expert_parallel_degree > 1:
+                    if self.moe_sharding_parallel_degree >= 1 and self.expert_parallel_degree > 1:
                         if is_context_parallel_supported():
                             order = ["sharding", "moe_sharding", "pp", "sep", "cp", "dp", "ep", "mp"]
                         else:
@@ -1429,8 +1429,6 @@ def is_context_parallel_supported():
                         "mp_degree": self.tensor_parallel_degree,
                         "pp_degree": self.pipeline_parallel_degree,
                         "sharding_degree": self.sharding_parallel_degree,
-                        "moe_sharding_degree": self.moe_sharding_parallel_degree,
-                        "ep_degree": self.expert_parallel_degree,
                         "sep_degree": self.sep_parallel_degree
                         if self.sep_parallel_degree > 1
                         else self.context_parallel_degree,
@@ -1442,11 +1440,16 @@ def is_context_parallel_supported():
                         "mp_degree": self.tensor_parallel_degree,
                         "pp_degree": self.pipeline_parallel_degree,
                         "sharding_degree": self.sharding_parallel_degree,
-                        "moe_sharding_degree": self.moe_sharding_parallel_degree,
-                        "ep_degree": self.expert_parallel_degree,
                         "order": order,
                     }
 
+                if self.expert_parallel_degree > 1:
+                    assert (
+                        self.use_expert_parallel is True and self.moe_sharding_parallel_degree >= 0
+                    ), f"invalid expert_parallel_degree {self.expert_parallel_degree} and use_expert_paralle:{self.use_expert_parallel}."
+                    hybrid_configs["ep_degree"] = self.expert_parallel_degree
+                    hybrid_configs["moe_sharding_degree"] = self.moe_sharding_parallel_degree
+
                 try:
                     if self.split_norm_comm:
                         hybrid_configs["split_norm_comm"] = True