auto_parallel add cp&sep strategy (#73195)

* add auto_parallel cp and sep strategy;

* add auto_parallel cp and sep strategy;

* add auto_parallel cp and sep strategy;

* fix sep and cp;

* fix sep and cp;

* fix sep and cp;

* fix ci

* fix ci

* fix ci

* add sep acc align test

* fix ci ut

* fix ci ut

* fix ci ut

* fix pr conflict

* fix intermediate api

* fix intermediate api

---------

Co-authored-by: andsonder <[email protected]>

Author: danleifeng

python/paddle/distributed/__init__.py

@@ -48,6 +48,10 @@
 )
 from .auto_parallel.high_level_api import to_distributed
 from .auto_parallel.interface import get_mesh, set_mesh
+from .auto_parallel.intermediate.context_parallel import (
+    ContextParallel,
+    PrepareContextParallel,
+)
 from .auto_parallel.intermediate.parallelize import parallelize
 from .auto_parallel.intermediate.pipeline_parallel import SplitPoint
 from .auto_parallel.intermediate.tensor_parallel import (
@@ -219,4 +223,6 @@
     "get_mesh",
     "to_distributed",
     "ConvParallel",
+    "ContextParallel",
+    "PrepareContextParallel",
 ]

python/paddle/distributed/auto_parallel/__init__.py

@@ -24,6 +24,7 @@
 )
 from .process_mesh import ProcessMesh  # noqa: F401
 from .random import parallel_manual_seed  # noqa: F401
+from .ring_attention import RingFlashAttention  # noqa: F401
 from .ring_conv import RingConv2d  # noqa: F401
 from .static.engine import Engine  # noqa: F401
 from .strategy import Strategy  # noqa: F401