We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent b31d7f9 commit 430741fCopy full SHA for 430741f
nemo/tron/config.py
@@ -115,9 +115,6 @@ class GPTDatasetConfig(MCoreGPTDatasetConfig):
115
data_sharding: bool = True
116
"""Disable data sharding."""
117
118
- create_attention_mask_in_dataloader: bool = True
119
- """If set, do not create attention_masks in dataloader."""
120
-
121
def __post_init__(self) -> None:
122
super(MCoreGPTDatasetConfig, self).__post_init__()
123
nemo/tron/llm/utils.py
@@ -76,7 +76,7 @@ def _broadcast(item):
76
dtype=torch.float32,
77
device=torch.cuda.current_device(),
78
)
79
- if cfg.dataset_config.create_attention_mask_in_dataloader:
+ if cfg.dataset_config.create_attention_mask:
80
attention_mask = torch.empty(
81
(
82
mbs,
0 commit comments