DS has poor default

thomasw21 · thomasw21 · commit 04cdc2e02704 · 2022-06-27T20:48:54.000+02:00
diff --git a/megatron/model/shared_t5_model.py b/megatron/model/shared_t5_model.py
@@ -83,6 +83,7 @@ def _to_16bit(inputs):
                     layer_type=LayerType.encoder,
                     layer_number=layer_idx,
                     self_attn_mask_type=AttnMaskType.causal,
+                    tied_weight_attr=None,
                     tied_weight_attrs=["self_attention", "mlp"]
                 ))
 
@@ -108,6 +109,7 @@ def _to_16bit(inputs):
                     layer_number=layer_idx,
                     layer_type=LayerType.decoder,
                     self_attn_mask_type=AttnMaskType.padding,
+                    tied_weight_attr=None,
                     tied_weight_attrs=["self_attention", "mlp"]
                 )
             )

Original file line number	Diff line number	Diff line change
`@@ -83,6 +83,7 @@ def _to_16bit(inputs):`
`83`	`83`	`layer_type=LayerType.encoder,`
`84`	`84`	`layer_number=layer_idx,`
`85`	`85`	`self_attn_mask_type=AttnMaskType.causal,`
	`86`	`+ tied_weight_attr=None,`
`86`	`87`	`tied_weight_attrs=["self_attention", "mlp"]`
`87`	`88`	`))`
`88`	`89`
`@@ -108,6 +109,7 @@ def _to_16bit(inputs):`
`108`	`109`	`layer_number=layer_idx,`
`109`	`110`	`layer_type=LayerType.decoder,`
`110`	`111`	`self_attn_mask_type=AttnMaskType.padding,`
	`112`	`+ tied_weight_attr=None,`
`111`	`113`	`tied_weight_attrs=["self_attention", "mlp"]`
`112`	`114`	`)`
`113`	`115`	`)`