cherry-pick PR 71507 to release/3.0 (#71732)

* initial commit for flagcx integration

* fix compilation issue related to dynamic loader

* fix compilation issue

* fix bugs and test distributed api functions

* set WITH_FLAGCX flag to off by defalut

* fix compilation issue when not compiling with flagcx

* add unit test and remove redundant code

* remove redundant code in comm_context_manager.cc

* remove debug log

* remove unit tests

* restore test file format

* modify code according review comments

* fix code format

* fix code format

Author: mikethegoblin

CMakeLists.txt

@@ -313,6 +313,7 @@ option(
   OFF)
 option(WITH_CINN "Compile PaddlePaddle with CINN" OFF)
 option(WITH_NCCL "Compile PaddlePaddle with NCCL support" ON)
+option(WITH_FLAGCX "Compile PaddlePaddle with FLAGCX support" OFF)
 option(WITH_RCCL "Compile PaddlePaddle with RCCL support" ON)
 option(WITH_XPU_BKCL "Compile PaddlePaddle with BAIDU KUNLUN XPU BKCL" OFF)
 option(WITH_CRYPTO "Compile PaddlePaddle with crypto support" ON)
@@ -538,6 +539,11 @@ else()
   endif()
 endif()
 
+if(WITH_FLAGCX)
+  add_definitions("-DPADDLE_WITH_FLAGCX")
+  include(flagcx)
+endif()
+
 if(WITH_HETERPS AND WITH_PSLIB)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
 endif()

cmake/flagcx.cmake

@@ -0,0 +1,24 @@
+set(CMAKE_FIND_DEBUG_MODE ON)
+# flagcx.cmake
+if(NOT WITH_FLAGCX)
+  return()
+endif()
+
+if(WITH_FLAGCX)
+  set(FLAGCX_ROOT
+      $ENV{FLAGCX_ROOT}
+      CACHE PATH "FLAGCX_ROOT")
+  message(STATUS "FLAGCX_ROOT is ${FLAGCX_ROOT}")
+  find_path(
+    FLAGCX_INCLUDE_DIR flagcx.h
+    PATHS ${FLAGCX_ROOT}/flagcx/include
+    NO_DEFAULT_PATH)
+  message(STATUS "FLAGCX_INCLUDE_DIR is ${FLAGCX_INCLUDE_DIR}")
+  include_directories(SYSTEM ${FLAGCX_INCLUDE_DIR})
+  set(FLAGCX_LIB
+      "${FLAGCX_ROOT}/build/lib/libflagcx.so"
+      CACHE FILEPATH "flagcx library." FORCE)
+  generate_dummy_static_lib(LIB_NAME "flagcx" GENERATOR "flagcx.cmake")
+  target_link_libraries(flagcx ${FLAGCX_LIB})
+  message(STATUS "FLAGCX_LIB is ${FLAGCX_LIB}")
+endif()

paddle/common/flags.cc

@@ -1261,6 +1261,19 @@ PHI_DEFINE_EXPORTED_bool(multi_node_sample_use_gpu_table,
 PHI_DEFINE_EXPORTED_bool(nccl_blocking_wait, false, "nccl blocking wait");
 #endif
 
+/**
+ * ProcessGroupFlagCX related FLAG
+ * Name: flagcx_blocking_wait
+ * Since Version:
+ * Value Range: bool, default=false
+ * Example:
+ * Note: nccl blocking wait.
+ * blocks host thread until collective operation completes
+ */
+#if defined(PADDLE_WITH_FLAGCX)
+PHI_DEFINE_EXPORTED_bool(flagcx_blocking_wait, false, "flagcx blocking wait");
+#endif
+
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 PHI_DEFINE_EXPORTED_bool(benchmark_nccl,
                          false,
@@ -1766,6 +1779,13 @@ PHI_DEFINE_EXPORTED_string(
     "For instance, /usr/local/cuda/lib64. If default, "
     "dlopen will search cuda from LD_LIBRARY_PATH");
 
+PHI_DEFINE_EXPORTED_string(
+    flagcx_dir,  // NOLINT
+    "",
+    "Specify path for loading libflagcx.so. For instance, "
+    "For instance, /usr/local/flagcx/lib. If default, "
+    "dlopen will search flagcx from LD_LIBRARY_PATH");
+
 PHI_DEFINE_EXPORTED_string(cupti_dir,
                            "",
                            "Specify path for loading cupti.so.");  // NOLINT

paddle/fluid/distributed/collective/CMakeLists.txt

@@ -36,6 +36,13 @@ if(WITH_NCCL OR WITH_RCCL)
 
 endif()
 
+if(WITH_FLAGCX)
+  cc_library(
+    process_group_flagcx
+    SRCS process_group_flagcx.cc common.cc
+    DEPS process_group phi)
+endif()
+
 if(WITH_XPU_BKCL)
   cc_library(
     process_group_bkcl
@@ -66,6 +73,9 @@ set(COMM_UTILS_DEPS process_group)
 if(WITH_NCCL OR WITH_RCCL)
   set(COMM_UTILS_DEPS ${PROCESS_GROUP_UTILS_DEPS} process_group_nccl)
 endif()
+if(WITH_FLAGCX)
+  set(COMM_UTILS_DEPS ${PROCESS_GROUP_UTILS_DEPS} process_group_flagcx)
+endif()
 if(WITH_CUSTOM_DEVICE)
   set(COMM_UTILS_DEPS ${PROCESS_GROUP_UTILS_DEPS} process_group_custom)
 endif()