Skip to content

Commit 4259c23

Browse files
committed
Fix detokenization regression
1 parent 13decb8 commit 4259c23

File tree

1 file changed

+1
-1
lines changed

1 file changed

+1
-1
lines changed

router/src/decoder.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ impl Decoder {
101101
Some(Metaspace(_) | WordPiece(_) | Sequence(_)) => {
102102
// For these, the first token in the sequence is treated differently,
103103
// so we add and then strip a placeholder token.
104-
let ids = [from_ref(&0), ids].concat();
104+
let ids = [from_ref(&self.single_tok_id), ids].concat();
105105
let result = self.decode_full(&*ids)?;
106106
let mut text = result.strip_prefix(&self.single_tok).ok_or_else(
107107
|| DetokenizationError("Unexpected".into())

0 commit comments

Comments
 (0)