[Cherry-Pick] support context parallel (#74201) (#74983)

* [Cherry-Pick] support context parallel (#74201)

* refine

* refine

* refine

* add ut

* fix no cp

* refine ut

* fix dense topo

Author: umiswing

paddle/fluid/framework/distributed_strategy.proto

@@ -137,6 +137,12 @@ message HybridConfig {
   optional EpConfig ep_configs = 17;
   optional MoeShardingConfig moe_sharding_configs = 18;
   optional DefaultCommGroupConfig default_comm_group_configs = 19;
+  optional int32 cp_degree = 20 [ default = 1 ];
+  optional int32 cp_sharding_degree = 21 [ default = 1 ];
+  optional CpConfig cp_configs = 22;
+  optional CpShardingConfig cp_sharding_configs = 23;
+  optional DpCpConfig dp_cp_configs = 24;
+  optional CpMpConfig cp_mp_configs = 25;
 }
 
 message AMPConfig {
@@ -502,6 +508,22 @@ message MoeShardingConfig {
   optional NCCLConfig check_nccl_config = 2;
 }
 
+message CpConfig {
+  optional NCCLConfig nccl_config = 1;
+}
+
+message CpShardingConfig {
+  optional NCCLConfig nccl_config = 1;
+}
+
+message DpCpConfig {
+  optional NCCLConfig nccl_config = 1;
+}
+
+message CpMpConfig {
+  optional NCCLConfig nccl_config = 1;
+}
+
 message DefaultCommGroupConfig {
   optional NCCLConfig nccl_config = 1;
 }

python/paddle/distributed/fleet/base/distributed_strategy.py

@@ -102,6 +102,7 @@ class _HybridConfig(TypedDict, total=False):
         mp_degree: int
         pp_degree: int
         sep_degree: int
+        cp_degree: int
         sharding_degree: int
         order: list[str]
 
@@ -325,6 +326,7 @@ def __init__(self) -> None:
             'pp',
             'sharding',
             'sep',
+            'cp',
             'mp',
         ]
         self.sync_param_name: list[str] = ["embedding", "layer_norm", ".b_"]
@@ -1907,6 +1909,7 @@ def hybrid_configs(self) -> _HybridConfig:
 
             **pp_degree(int)**: set number of GPUs in a pipeline parallel group. Default 1
             **sep_degree(int)**: set number of GPUs in a sep parallel group. Default 1
+            **cp_degree(int)**: set number of GPUs in a context parallel group. Default 1
             **sharding_degree(int)**: set number of GPUs in a sharding parallel group. Default 1
             **order(list(string))**: set hybrid parallel dimensions, the order is from outside to inside. Default ['dp','pp','sharding','sep', 'mp']