Skip to content

Commit 83b96f4

Browse files
authored
fix truncation_strategy for tokenizer (#1615)
1 parent eb1dd6d commit 83b96f4

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

paddlenlp/transformers/tokenizer_utils.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -616,7 +616,7 @@ def __call__(self,
616616
stride=stride,
617617
is_split_into_words=is_split_into_words,
618618
pad_to_max_seq_len=pad_to_max_seq_len,
619-
truncation_strategy="longest_first",
619+
truncation_strategy=truncation_strategy,
620620
return_position_ids=return_position_ids,
621621
return_token_type_ids=return_token_type_ids,
622622
return_attention_mask=return_attention_mask,
@@ -629,7 +629,7 @@ def __call__(self,
629629
text_pair=text_pair,
630630
max_seq_len=max_seq_len,
631631
pad_to_max_seq_len=pad_to_max_seq_len,
632-
truncation_strategy="longest_first",
632+
truncation_strategy=truncation_strategy,
633633
return_position_ids=return_position_ids,
634634
return_token_type_ids=return_token_type_ids,
635635
return_attention_mask=return_attention_mask,

0 commit comments

Comments
 (0)