fix nccl dist train bug

yi.wu · yi.wu · commit d76ebd7853e3 · 2018-06-13T11:23:56.000+08:00
diff --git a/paddle/fluid/operators/gen_nccl_id_op.cc b/paddle/fluid/operators/gen_nccl_id_op.cc
@@ -67,6 +67,10 @@ class GenNCCLIdOp : public framework::OperatorBase {
       client->AsyncSendVar(ep, dev_ctx, *scope, NCCL_ID_VARNAME);
     }
     client->Wait();
+    for (auto& ep : endpoint_list) {
+      client->AsyncSendBatchBarrier(ep);
+    }
+    client->Wait();
     VLOG(3) << "sending completed...";
   }
 

Original file line number	Diff line number	Diff line change
`@@ -67,6 +67,10 @@ class GenNCCLIdOp : public framework::OperatorBase {`
`67`	`67`	`client->AsyncSendVar(ep, dev_ctx, *scope, NCCL_ID_VARNAME);`
`68`	`68`	`}`
`69`	`69`	`client->Wait();`
	`70`	`+ for (auto& ep : endpoint_list) {`
	`71`	`+ client->AsyncSendBatchBarrier(ep);`
	`72`	`+ }`
	`73`	`+ client->Wait();`
`70`	`74`	`VLOG(3) << "sending completed...";`
`71`	`75`	`}`
`72`	`76`