We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 13decb8 commit 4259c23Copy full SHA for 4259c23
router/src/decoder.rs
@@ -101,7 +101,7 @@ impl Decoder {
101
Some(Metaspace(_) | WordPiece(_) | Sequence(_)) => {
102
// For these, the first token in the sequence is treated differently,
103
// so we add and then strip a placeholder token.
104
- let ids = [from_ref(&0), ids].concat();
+ let ids = [from_ref(&self.single_tok_id), ids].concat();
105
let result = self.decode_full(&*ids)?;
106
let mut text = result.strip_prefix(&self.single_tok).ok_or_else(
107
|| DetokenizationError("Unexpected".into())
0 commit comments