fix: Sanitize hidden Unicode characters from user and tool inputs (#2435)

* add a sanitize_unicode_tag method

* add ut

* clean execute_bash result

* fmt

* exclude fffd when fileter

* import saitize func

* fix typo

Author: evanliu048

crates/chat-cli/src/cli/chat/cli/usage.rs

@@ -106,7 +106,7 @@ impl UsageArgs {
                 )),
                 // Context files
                 style::SetForegroundColor(Color::DarkCyan),
-                // add a nice visual to mimic "tiny" progress, so the overral progress bar doesn't look too
+                // add a nice visual to mimic "tiny" progress, so the overrall progress bar doesn't look too
                 // empty
                 style::Print("|".repeat(if context_width == 0 && *context_token_count > 0 {
                     1

crates/chat-cli/src/cli/chat/mod.rs

@@ -138,6 +138,7 @@ use crate::cli::chat::cli::prompts::{
     GetPromptError,
     PromptsSubcommand,
 };
+use crate::cli::chat::util::sanitize_unicode_tags;
 use crate::database::settings::Setting;
 use crate::mcp_client::Prompt;
 use crate::os::Os;
@@ -1585,7 +1586,7 @@ impl ChatSession {
 
     async fn handle_input(&mut self, os: &mut Os, mut user_input: String) -> Result<ChatState, ChatError> {
         queue!(self.stderr, style::Print('\n'))?;
-
+        user_input = sanitize_unicode_tags(&user_input);
         let input = user_input.trim();
 
         // handle image path

crates/chat-cli/src/cli/chat/tools/execute/mod.rs

@@ -13,6 +13,7 @@ use crate::cli::agent::{
     Agent,
     PermissionEvalResult,
 };
+use crate::cli::chat::sanitize_unicode_tags;
 use crate::cli::chat::tools::{
     InvokeOutput,
     MAX_TOOL_RESPONSE_SIZE,
@@ -120,10 +121,13 @@ impl ExecuteCommand {
 
     pub async fn invoke(&self, output: &mut impl Write) -> Result<InvokeOutput> {
         let output = run_command(&self.command, MAX_TOOL_RESPONSE_SIZE / 3, Some(output)).await?;
+        let clean_stdout = sanitize_unicode_tags(&output.stdout);
+        let clean_stderr = sanitize_unicode_tags(&output.stderr);
+
         let result = serde_json::json!({
             "exit_status": output.exit_status.unwrap_or(0).to_string(),
-            "stdout": output.stdout,
-            "stderr": output.stderr,
+            "stdout": clean_stdout,
+            "stderr": clean_stderr,
         });
 
         Ok(InvokeOutput {

crates/chat-cli/src/cli/chat/tools/fs_read.rs

@@ -37,13 +37,16 @@ use crate::cli::agent::{
     Agent,
     PermissionEvalResult,
 };
-use crate::cli::chat::CONTINUATION_LINE;
 use crate::cli::chat::tools::display_purpose;
 use crate::cli::chat::util::images::{
     handle_images_from_paths,
     is_supported_image_type,
     pre_process,
 };
+use crate::cli::chat::{
+    CONTINUATION_LINE,
+    sanitize_unicode_tags,
+};
 use crate::os::Os;
 
 #[derive(Debug, Clone, Deserialize)]
@@ -451,6 +454,7 @@ impl FsLine {
         debug!(?path, "Reading");
         let file_bytes = os.fs.read(&path).await?;
         let file_content = String::from_utf8_lossy(&file_bytes);
+        let file_content = sanitize_unicode_tags(&file_content);
         let line_count = file_content.lines().count();
         let (start, end) = (
             convert_negative_index(line_count, self.start_line()),
@@ -559,6 +563,7 @@ impl FsSearch {
 
         let file_bytes = os.fs.read(&file_path).await?;
         let file_content = String::from_utf8_lossy(&file_bytes);
+        let file_content = sanitize_unicode_tags(&file_content);
         let lines: Vec<&str> = LinesWithEndings::from(&file_content).collect();
 
         let mut results = Vec::new();

crates/chat-cli/src/cli/chat/util/mod.rs

@@ -59,6 +59,48 @@ pub fn animate_output(output: &mut impl Write, bytes: &[u8]) -> Result<(), ChatE
     Ok(())
 }
 
+/// Returns `true` if the character is from an invisible or control Unicode range
+/// that is considered unsafe for LLM input. These rarely appear in normal input,
+/// so stripping them is generally safe.
+/// The replacement character U+FFFD (�) is preserved to indicate invalid bytes.
+fn is_hidden(c: char) -> bool {
+    match c {
+        '\u{E0000}'..='\u{E007F}' |     // TAG characters (used for hidden prompts)  
+        '\u{200B}'..='\u{200F}'  |      // zero-width space, ZWJ, ZWNJ, RTL/LTR marks  
+        '\u{2028}'..='\u{202F}'  |      // line / paragraph separators, narrow NB-SP  
+        '\u{205F}'..='\u{206F}'  |      // format control characters  
+        '\u{FFF0}'..='\u{FFFC}'  |
+        '\u{FFFE}'..='\u{FFFF}'   // Specials block (non-characters) 
+        => true,
+        _ => false,
+    }
+}
+
+/// Remove hidden / control characters from `text`.
+///
+/// * `text`   –  raw user input or file content
+///
+/// The function keeps things **O(n)** with a single allocation and logs how many
+/// characters were dropped. 400 KB worst-case size ⇒ sub-millisecond runtime.
+pub fn sanitize_unicode_tags(text: &str) -> String {
+    let mut removed = 0;
+    let out: String = text
+        .chars()
+        .filter(|&c| {
+            let bad = is_hidden(c);
+            if bad {
+                removed += 1;
+            }
+            !bad
+        })
+        .collect();
+
+    if removed > 0 {
+        tracing::debug!("Detected and removed {} hidden chars", removed);
+    }
+    out
+}
+
 /// Play the terminal bell notification sound
 pub fn play_notification_bell(requires_confirmation: bool) {
     // Don't play bell for tools that don't require confirmation
@@ -249,4 +291,39 @@ mod tests {
         }
         assert_eq!(files.len(), 1);
     }
+    #[test]
+    fn is_hidden_recognises_all_ranges() {
+        let samples = ['\u{E0000}', '\u{200B}', '\u{2028}', '\u{205F}', '\u{FFF0}'];
+
+        for ch in samples {
+            assert!(is_hidden(ch), "char U+{:X} should be hidden", ch as u32);
+        }
+
+        for ch in ['a', '你', '\u{03A9}'] {
+            assert!(!is_hidden(ch), "char {:?} should NOT be hidden", ch);
+        }
+    }
+
+    #[test]
+    fn sanitize_keeps_visible_text_intact() {
+        let visible = "Rust 🦀 > C";
+        assert_eq!(sanitize_unicode_tags(visible), visible);
+    }
+
+    #[test]
+    fn sanitize_handles_large_mixture() {
+        let visible_block = "abcXYZ";
+        let hidden_block = "\u{200B}\u{E0000}";
+        let mut big_input = String::new();
+        for _ in 0..50_000 {
+            big_input.push_str(visible_block);
+            big_input.push_str(hidden_block);
+        }
+
+        let result = sanitize_unicode_tags(&big_input);
+
+        assert_eq!(result.len(), 50_000 * visible_block.len());
+
+        assert!(result.chars().all(|c| !is_hidden(c)));
+    }
 }