Merge pull request #439 from yinhaofeng/collective_train_2

fix static auc

Author: seemingwang

tools/static_trainer.py

@@ -131,7 +131,7 @@ def main(args):
 
         epoch_begin = time.time()
         if use_auc:
-            reset_auc(auc_num)
+            reset_auc(use_fleet, auc_num)
         if reader_type == 'DataLoader':
             fetch_batch_var, step_num = dataloader_train(
                 epoch_id, train_dataloader, input_data_names, fetch_vars, exe,
@@ -234,6 +234,7 @@ def dataloader_train(epoch_id, train_dataloader, input_data_names, fetch_vars,
             program=paddle.static.default_main_program(),
             feed=dict(zip(input_data_names, batch_data)),
             fetch_list=[var for _, var in fetch_vars.items()])
+        # print(paddle.fluid.global_scope().find_var("_generated_var_2").get_tensor())
         train_run_cost += time.time() - train_start
         total_samples += batch_size
         if batch_id % print_interval == 0:

tools/utils/utils_single.py

@@ -130,7 +130,7 @@ def load_yaml(yaml_file, other_part=None):
     return running_config
 
 
-def reset_auc(auc_num=1):
+def reset_auc(use_fleet=False, auc_num=1):
     # for static clear auc
     auc_var_name = []
     for i in range(auc_num * 4):
@@ -143,5 +143,9 @@ def reset_auc(auc_num=1):
         tensor = param.get_tensor()
         if param:
             tensor_array = np.zeros(tensor._get_dims()).astype("int64")
-            tensor.set(tensor_array, paddle.CPUPlace())
+            if use_fleet:
+                trainer_id = paddle.distributed.get_rank()
+                tensor.set(tensor_array, paddle.CUDAPlace(trainer_id))
+            else:
+                tensor.set(tensor_array, paddle.CPUPlace())
             logger.info("AUC Reset To Zero: {}".format(name))