Merge pull request #1414 from wangxicoding/fix_static_bert_amp_nan

fix static bert amp nan

Author: wawltor

examples/language_model/bert/static/dataset.py

@@ -125,7 +125,7 @@ def __getitem__(self, index):
         # TODO: whether to use reversed mask by changing 1s and 0s to be
         # consistent with nv bert
         input_mask = (1 - np.reshape(
-            input_mask.astype(np.float32), [1, 1, input_mask.shape[0]])) * -1e9
+            input_mask.astype(np.float32), [1, 1, input_mask.shape[0]])) * -1e4
 
         index = self.max_pred_length
         # store number of  masked tokens in index

examples/language_model/bert/static/run_pretrain.py

@@ -136,7 +136,7 @@ def parse_args():
     parser.add_argument(
         "--scale_loss",
         type=float,
-        default=1.0,
+        default=2**15,
         help="The value of scale_loss for fp16.")
     parser.add_argument(
         "--use_pure_fp16",