From 278f3b9f2a7f569ad16cddc284e7cf452ecdd856 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Thu, 21 Aug 2025 14:28:16 +0800
Subject: [PATCH] common : fix incorrect print of non-ascii characters in the
 logging

Signed-off-by: Jie Fu <jiefu@tencent.com>
---
 common/common.cpp | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/common/common.cpp b/common/common.cpp
index 67dd5404fff90..decabcc2ed327 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -558,13 +558,6 @@ std::string string_from(const struct llama_context * ctx, const std::vector<llam
 
         auto detokenized = common_token_to_piece(ctx, token);
 
-        detokenized.erase(
-            std::remove_if(
-                detokenized.begin(),
-                detokenized.end(),
-                [](const unsigned char c) { return !std::isprint(c); }),
-            detokenized.end());
-
         buf << "'" << detokenized << "'"
             << ":" << std::to_string(token);
     }
@@ -589,13 +582,6 @@ std::string string_from(const struct llama_context * ctx, const struct llama_bat
 
         auto detokenized = common_token_to_piece(ctx, batch.token[i]);
 
-        detokenized.erase(
-                std::remove_if(
-                    detokenized.begin(),
-                    detokenized.end(),
-                    [](const unsigned char c) { return !std::isprint(c); }),
-                detokenized.end());
-
         buf << "\n"          << std::to_string(i)
             << ", token '"   << detokenized << "'"
             << ", pos "      << std::to_string(batch.pos[i])