barrier_worker

Author: esythan

tools/static_gpubox_trainer.py

@@ -179,6 +179,7 @@ def run_worker(self):
                     self.exe, model_dir,
                     [feed.name for feed in self.input_data],
                     self.inference_target_var)
+            fleet.barrier_worker()
             self.reader.release_memory()
             self.PSGPU.end_pass()
             logger.info("finish {} epoch training....".format(epoch))

tools/static_ps_online_trainer.py

@@ -407,6 +407,7 @@ def run_online_worker(self):
                         xbox_base_key=model_base_key,
                         train_local=self.train_local,
                         client=self.hadoop_client)
+            fleet.barrier_worker()
             day = get_next_day(day)
 
     def dataset_train_loop(self, cur_dataset, day, pass_index,

tools/static_ps_trainer.py

@@ -166,6 +166,7 @@ def run_worker(self):
                         model_dir,
                         [feed.name for feed in self.inference_feed_var],
                         [self.inference_target_var], self.exe)
+            fleet.barrier_worker()
 
         if reader_type == "InmemoryDataset":
             self.reader.release_memory()