Fixed break on unk error

Author: kinkel-ralf

embeddings/src/main.rs

@@ -137,11 +137,19 @@ fn main() -> Result<()> {
     for (i, token_line) in tokens_lines_list.iter().enumerate() {
         eprintln!("Prompt {i}");
         for token in token_line {
-            eprintln!(" {} --> {}", token, model.token_to_str(*token)?);
+            // Attempt to convert token to string and print it; if it fails, print the token instead
+            match model.token_to_str(*token) {
+                Ok(token_str) => eprintln!("Token string: {}", token_str),
+                Err(e) => {
+                    eprintln!("Failed to convert token to string, error: {}", e);
+                    eprintln!("Token value: {}", token);
+                }
+            }
         }
         eprintln!();
     }
 
+    
     std::io::stderr().flush()?;
 
     // create a llama_batch with the size of the context