fix cherrpicked commits

Author: MarcusDunn

embeddings/src/main.rs

@@ -20,7 +20,7 @@ use llama_cpp_2::ggml_time_us;
 use llama_cpp_2::llama_backend::LlamaBackend;
 use llama_cpp_2::llama_batch::LlamaBatch;
 use llama_cpp_2::model::params::LlamaModelParams;
-use llama_cpp_2::model::AddBos;
+use llama_cpp_2::model::{AddBos, Special};
 use llama_cpp_2::model::LlamaModel;
 
 #[derive(clap::Parser, Debug, Clone)]
@@ -138,7 +138,7 @@ fn main() -> Result<()> {
         eprintln!("Prompt {i}");
         for token in token_line {
             // Attempt to convert token to string and print it; if it fails, print the token instead
-            match model.token_to_str(*token) {
+            match model.token_to_str(*token, Special::Tokenize) {
                 Ok(token_str) => eprintln!(" {} --> {}", token, token_str),
                 Err(e) => {
                     eprintln!("Failed to convert token to string, error: {}", e);

llama-cpp-2/src/model.rs

@@ -122,8 +122,8 @@ impl LlamaModel {
     /// # Errors
     ///
     /// See [`TokenToStringError`] for more information.
-    pub fn token_to_bytes(&self, token: LlamaToken) -> Result<Vec<u8>, TokenToStringError> {
-        self.token_to_bytes_with_size(token, 32)
+    pub fn token_to_bytes(&self, token: LlamaToken, special: Special) -> Result<Vec<u8>, TokenToStringError> {
+        self.token_to_bytes_with_size(token, 32, special)
     }
 
     /// Convert a vector of tokens to a single string.
@@ -248,7 +248,7 @@ impl LlamaModel {
         buffer_size: usize,
         special: Special,
     ) -> Result<String, TokenToStringError> {
-        let bytes = self.token_to_bytes_with_size(token, buffer_size)?;
+        let bytes = self.token_to_bytes_with_size(token, buffer_size, special)?;
         Ok(String::from_utf8(bytes)?)
     }
 
@@ -270,6 +270,7 @@ impl LlamaModel {
         &self,
         token: LlamaToken,
         buffer_size: usize,
+        special: Special,
     ) -> Result<Vec<u8>, TokenToStringError> {
         if token == self.token_nl() {
             return Ok(String::from("\n").into_bytes());

simple/src/main.rs

@@ -259,7 +259,7 @@ either reduce n_len or increase n_ctx"
                 break;
             }
 
-            let output_bytes = model.token_to_bytes(new_token_id)?;
+            let output_bytes = model.token_to_bytes(new_token_id, Special::Tokenize)?;
             // use `Decoder.decode_to_string()` to avoid the intermediate buffer
             let mut output_string = String::with_capacity(32);
             let _decode_result = decoder.decode_to_string(&output_bytes, &mut output_string, false);