|
| 1 | +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | + |
| 14 | +from __future__ import print_function |
| 15 | +from __future__ import division |
| 16 | +import os |
| 17 | + |
| 18 | +import paddle.fluid as fluid |
| 19 | +from paddle.fluid import core, unique_name |
| 20 | +from ..base.private_helper_function import wait_server_ready |
| 21 | +from .meta_optimizer_base import MetaOptimizerBase |
| 22 | +from .common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY, CollectiveHelper, is_loss_grad_op, is_backward_op, is_optimizer_op |
| 23 | + |
| 24 | + |
| 25 | +class RawProgramOptimizer(MetaOptimizerBase): |
| 26 | + def __init__(self, optimizer): |
| 27 | + super(RawProgramOptimizer, self).__init__(optimizer) |
| 28 | + self.inner_opt = optimizer |
| 29 | + self.meta_optimizers_white_list = [ |
| 30 | + "RecomputeOptimizer", |
| 31 | + "AMPOptimizer", |
| 32 | + ] |
| 33 | + self.meta_optimizers_black_list = ["GraphExecutionOptimizer", ] |
| 34 | + self.global_ring_id = 0 |
| 35 | + |
| 36 | + def _set_basic_info(self, loss, role_maker, user_defined_optimizer, |
| 37 | + user_defined_strategy): |
| 38 | + super(RawProgramOptimizer, self)._set_basic_info( |
| 39 | + loss, role_maker, user_defined_optimizer, user_defined_strategy) |
| 40 | + self.without_graph_optimization = user_defined_strategy.without_graph_optimization |
| 41 | + |
| 42 | + def _can_apply(self): |
| 43 | + if not self.role_maker._is_collective: |
| 44 | + return False |
| 45 | + |
| 46 | + if self.without_graph_optimization == True: |
| 47 | + return True |
| 48 | + return False |
| 49 | + |
| 50 | + def _disable_strategy(self, dist_strategy): |
| 51 | + dist_strategy.without_graph_optimization = False |
| 52 | + |
| 53 | + def _enable_strategy(self, dist_strategy, context): |
| 54 | + dist_strategy.without_graph_optimization = True |
| 55 | + |
| 56 | + def _broadcast_params(self, ring_id): |
| 57 | + block = self.startup_program.global_block() |
| 58 | + param = None |
| 59 | + for param in block.iter_parameters(): |
| 60 | + if param.is_distributed: |
| 61 | + continue |
| 62 | + |
| 63 | + block.append_op( |
| 64 | + type='c_broadcast', |
| 65 | + inputs={'X': param}, |
| 66 | + outputs={'Out': param}, |
| 67 | + attrs={ |
| 68 | + 'ring_id': ring_id, |
| 69 | + 'root': 0, |
| 70 | + OP_ROLE_KEY: OpRole.Forward |
| 71 | + }) |
| 72 | + |
| 73 | + if not param: return # no parameter on this device |
| 74 | + block.append_op( |
| 75 | + type='c_sync_comm_stream', |
| 76 | + inputs={'X': param}, |
| 77 | + outputs={'Out': param}, |
| 78 | + attrs={'ring_id': ring_id, |
| 79 | + OP_ROLE_KEY: OpRole.Forward}) |
| 80 | + |
| 81 | + def _get_process_group_info(self): |
| 82 | + # global ring info |
| 83 | + self.global_endpoints = self.endpoints |
| 84 | + self.global_rank = self.rank |
| 85 | + self.global_nranks = self.nranks |
| 86 | + |
| 87 | + def _init_process_group(self): |
| 88 | + self._get_process_group_info() |
| 89 | + collective_helper = CollectiveHelper(self.role_maker, wait_port=False) |
| 90 | + # Create global ring for all gpus (ring_id = 0) |
| 91 | + collective_helper._init_communicator( |
| 92 | + self.startup_program, self.current_endpoint, self.global_endpoints, |
| 93 | + self.global_rank, self.global_ring_id, True, self.global_ring_id, |
| 94 | + True) |
| 95 | + self._broadcast_params(self.global_ring_id) |
| 96 | + |
| 97 | + def minimize_impl(self, |
| 98 | + loss, |
| 99 | + startup_program=None, |
| 100 | + parameter_list=None, |
| 101 | + no_grad_set=None): |
| 102 | + self.endpoints = self.role_maker._get_trainer_endpoints() |
| 103 | + self.current_endpoint = self.endpoints[self.role_maker._worker_index()] |
| 104 | + self.rank = self.role_maker._worker_index() |
| 105 | + self.nranks = self.role_maker._worker_num() |
| 106 | + if startup_program is None: |
| 107 | + startup_program = fluid.default_startup_program() |
| 108 | + self.startup_program = startup_program |
| 109 | + |
| 110 | + block = loss.block |
| 111 | + program = block.program |
| 112 | + self.main_program = program |
| 113 | + |
| 114 | + optimize_ops, params_grads = self.inner_opt.minimize( |
| 115 | + loss, startup_program, parameter_list, no_grad_set) |
| 116 | + if self.nranks == 1: |
| 117 | + return optimize_ops, params_grads |
| 118 | + self._init_process_group() |
| 119 | + |
| 120 | + self.main_program = program |
| 121 | + if self.nranks > 1: |
| 122 | + self._transpile_main_program(loss) |
| 123 | + return optimize_ops, params_grads |
| 124 | + |
| 125 | + def _transpile_main_program(self, loss): |
| 126 | + self._insert_loss_grad_ops(loss) |
| 127 | + self._insert_allreduce_ops() |
| 128 | + |
| 129 | + def _insert_loss_grad_ops(self, loss): |
| 130 | + """ |
| 131 | + In order to keep the learning rate consistent in different numbers of |
| 132 | + training workers, we scale the loss grad by the number of workers |
| 133 | + """ |
| 134 | + block = self.main_program.global_block() |
| 135 | + for idx, op in reversed(list(enumerate(block.ops))): |
| 136 | + if is_loss_grad_op(op): |
| 137 | + loss_grad_var = block.vars[op.output_arg_names[0]] |
| 138 | + block._insert_op( |
| 139 | + idx + 1, |
| 140 | + type='scale', |
| 141 | + inputs={'X': loss_grad_var}, |
| 142 | + outputs={'Out': loss_grad_var}, |
| 143 | + attrs={ |
| 144 | + 'scale': 1.0 / self.nranks, |
| 145 | + OP_ROLE_KEY: OpRole.Backward |
| 146 | + }) |
| 147 | + |
| 148 | + def _insert_allreduce_ops(self): |
| 149 | + block = self.main_program.global_block() |
| 150 | + ring_id = self.global_ring_id |
| 151 | + grad = None |
| 152 | + for idx, op in reversed(list(enumerate(block.ops))): |
| 153 | + if is_backward_op(op) and \ |
| 154 | + OP_ROLE_VAR_KEY in op.attr_names: |
| 155 | + op_role_var = op.attr(OP_ROLE_VAR_KEY) |
| 156 | + if len(op_role_var) == 0: |
| 157 | + continue |
| 158 | + assert len(op_role_var) % 2 == 0 |
| 159 | + offset = 1 |
| 160 | + for i in range(0, len(op_role_var), 2): |
| 161 | + param_name = op_role_var[i] |
| 162 | + param = block.var(param_name) |
| 163 | + grad_name = op_role_var[i + 1] |
| 164 | + grad = block.var(grad_name) |
| 165 | + if param.is_distributed: |
| 166 | + continue |
| 167 | + |
| 168 | + block._insert_op( |
| 169 | + idx + offset, |
| 170 | + type='c_sync_calc_stream', |
| 171 | + inputs={'X': grad}, |
| 172 | + outputs={'Out': grad}, |
| 173 | + attrs={OP_ROLE_KEY: OpRole.Backward, }) |
| 174 | + offset += 1 |
| 175 | + block._insert_op( |
| 176 | + idx + offset, |
| 177 | + type='c_allreduce_sum', |
| 178 | + inputs={'X': grad}, |
| 179 | + outputs={'Out': grad}, |
| 180 | + attrs={ |
| 181 | + 'ring_id': ring_id, |
| 182 | + OP_ROLE_KEY: OpRole.Backward |
| 183 | + }) |
| 184 | + |
| 185 | + if grad is None: |
| 186 | + return |
| 187 | + |
| 188 | + for idx, op in enumerate(block.ops): |
| 189 | + if is_optimizer_op(op): |
| 190 | + block._insert_op( |
| 191 | + idx, |
| 192 | + type='c_sync_comm_stream', |
| 193 | + inputs={'X': grad}, |
| 194 | + outputs={'Out': grad}, |
| 195 | + attrs={'ring_id': ring_id, |
| 196 | + OP_ROLE_KEY: OpRole.Backward}) |
| 197 | + break |
0 commit comments