Merge pull request #250 from rachiki/fix/handle_unk_tokens_emb

Fix/handle unk tokens emb

Author: MarcusDunn

embeddings/src/main.rs

@@ -137,7 +137,14 @@ fn main() -> Result<()> {
     for (i, token_line) in tokens_lines_list.iter().enumerate() {
         eprintln!("Prompt {i}");
         for token in token_line {
-            eprintln!(" {} --> {}", token, model.token_to_str(*token)?);
+            // Attempt to convert token to string and print it; if it fails, print the token instead
+            match model.token_to_str(*token) {
+                Ok(token_str) => eprintln!(" {} --> {}", token, token_str),
+                Err(e) => {
+                    eprintln!("Failed to convert token to string, error: {}", e);
+                    eprintln!("Token value: {}", token);
+                }
+            }
         }
         eprintln!();
     }