Skip to content

Commit 02a9dbe

Browse files
add missing is_train kwarg in tokenize_id (#187)
1 parent 79a10be commit 02a9dbe

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

eole/transforms/tokenize_id.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,10 +96,10 @@ def tokenize_string(self, string, side="src", is_train=False):
9696
return tokens
9797

9898
def apply(self, example, is_train=False, stats=None, **kwargs):
99-
src_tokens = self.tokenize_string(" ".join(example["src"]), side="src")
99+
src_tokens = self.tokenize_string(" ".join(example["src"]), side="src", is_train=is_train)
100100
example["src_ids"] = src_tokens
101101
if example.get("tgt", None) is not None:
102-
tgt_tokens = self.tokenize_string(" ".join(example["tgt"]), side="tgt")
102+
tgt_tokens = self.tokenize_string(" ".join(example["tgt"]), side="tgt", is_train=is_train)
103103
example["tgt_ids"] = tgt_tokens
104104
return example
105105

0 commit comments

Comments
 (0)