benchmark train

Author: yinhaofeng

models/recall/word2vec/benchmark/static_model.py

@@ -156,7 +156,8 @@ def net(self, inputs, is_infer=False):
 
         return self.metrics
 
-    def create_optimizer(self, strategy=None, pure_bf16=False):
+    def create_optimizer(self, strategy=None):
+        pure_bf16 = self.config.get("pure_bf16")
         lr = float(self.config.get("hyper_parameters.optimizer.learning_rate"))
         decay_rate = float(
             self.config.get("hyper_parameters.optimizer.decay_rate"))

tools/static_ps_trainer.py

@@ -26,6 +26,8 @@
 import warnings
 import logging
 import ast
+import numpy as np
+import struct
 
 __dir__ = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
@@ -60,6 +62,10 @@ def parse_args():
     return config
 
 
+def bf16_to_fp32(val):
+    return np.float32(struct.unpack('<f', struct.pack('<I', val << 16))[0])
+
+
 class Main(object):
     def __init__(self, config):
         self.metrics = {}
@@ -86,7 +92,7 @@ def run(self):
     def network(self):
         self.model = get_model(self.config)
         self.input_data = self.model.create_feeds()
-        self.inference_feed_var = self.model.create_feeds(is_infer=True)
+        self.inference_feed_var = self.model.create_feeds(is_infer=False)
         self.init_reader()
         self.metrics = self.model.net(self.input_data)
         self.inference_target_var = self.model.inference_target_var
@@ -212,7 +218,6 @@ def dataloader_train_loop(self, epoch):
         batch_id = 0
         train_run_cost = 0.0
         total_examples = 0
-        from paddle.fluid.tests.unittests.op_test import convert_uint16_to_float
         self.reader.start()
         while True:
             try:
@@ -232,7 +237,7 @@ def dataloader_train_loop(self, epoch):
                         metrics_string += "{}: {}, ".format(
                             var_name, fetch_var[var_idx]
                             if var_name != "LOSS" or not config['pure_bf16']
-                            else convert_uint16_to_float(fetch_var[var_idx]))
+                            else bf16_to_fp32(fetch_var[var_idx][0]))
                     profiler_string = ""
                     profiler_string += "avg_batch_cost: {} sec, ".format(
                         format((train_run_cost) / print_step, '.5f'))