File tree Expand file tree Collapse file tree 1 file changed +5
-1
lines changed
paddle/fluid/framework/details Expand file tree Collapse file tree 1 file changed +5
-1
lines changed Original file line number Diff line number Diff line change @@ -460,10 +460,14 @@ std::unique_ptr<ir::Graph> MultiDevSSAGraphBuilder::ApplyImpl(
460
460
use_gpu = nccl_ctxs_ != nullptr ;
461
461
#endif
462
462
463
+ // Insert broadcast operators principle:
464
+ // 1. Broadcast optimized parameters in Reduce strategy;
465
+ // 2. No need broadcast optimized parameters in AllReduce strategy because of
466
+ // the optimization sub-graph would be run on every GPU;
467
+ // 3. Allways broadcast received parameters in Distribute Training.
463
468
if ((use_gpu &&
464
469
strategy_.reduce_ == BuildStrategy::ReduceStrategy::kReduce ) ||
465
470
is_dist_train) {
466
- // allways broadcast receieved parameters for distributed training
467
471
for (size_t dev_id = 0 ; dev_id < bcast_var_name_set.size (); ++dev_id) {
468
472
auto &to_bcast_set = bcast_var_name_set[dev_id];
469
473
for (auto &bcast_name : to_bcast_set) {
You can’t perform that action at this time.
0 commit comments