Update turbomind communication library (#3736)

* relax fp8 tp requirement

* initial NVLS support

* refactor allocation & registration

* more nvls collectives

* semaphore v2

* allgather v2

* add broadcast experiments

* minor

* allreduce

* refactor

* per group semaphore

* add NCCL 2.27 window registration

* bf16

* minor

* fix lint

* fix lint

* minor

* guard sm90 & cu12 features

* disable multimem for world size < 4

* fix semaphore init

* add test for broadcast

* fix semaphore init

* disable low latency kernels

* enable low latency kernels

Author: lzhangzz

src/turbomind/comm/CMakeLists.txt

@@ -23,6 +23,6 @@ if (BUILD_MULTI_GPU)
     if (BUILD_TEST)
         add_executable(test_comm test_comm.cu)
         target_link_libraries(test_comm PRIVATE device_comm host_comm core pthread nvtx_utils)
-        target_compile_options(test_comm PRIVATE -O3 -march=native -mtune=native)
+        target_compile_options(test_comm PRIVATE -march=native -mtune=native)
     endif ()
 endif ()

src/turbomind/comm/cuda_ipc/CMakeLists.txt

@@ -7,7 +7,8 @@ add_library(cuda_ipc_comm STATIC
         allreduce.cu
         allgather.cu
         fused_allreduce.cu
-        fused_allreduce_ex.cu)
+        fused_allreduce_ex.cu
+        broadcast.cu)
 
 target_link_libraries(cuda_ipc_comm PRIVATE
         rms_norm