From: Jie Fu (傅杰) <redacted>
Date: Thu, 21 Aug 2025 08:54:34 +0000 (+0800)
Subject: common : fix incorrect print of non-ascii characters in the logging (#15466)
X-Git-Tag: upstream/0.0.6527~302
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=2f3dbffb17ef782edfd50e5a130cec6e8a7e47f8;p=pkg%2Fggml%2Fsources%2Fllama.cpp

common : fix incorrect print of non-ascii characters in the logging (#15466)

Signed-off-by: Jie Fu <redacted>
---

diff --git a/common/common.cpp b/common/common.cpp
index 67dd5404..decabcc2 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -558,13 +558,6 @@ std::string string_from(const struct llama_context * ctx, const std::vector<llam
 
         auto detokenized = common_token_to_piece(ctx, token);
 
-        detokenized.erase(
-            std::remove_if(
-                detokenized.begin(),
-                detokenized.end(),
-                [](const unsigned char c) { return !std::isprint(c); }),
-            detokenized.end());
-
         buf << "'" << detokenized << "'"
             << ":" << std::to_string(token);
     }
@@ -589,13 +582,6 @@ std::string string_from(const struct llama_context * ctx, const struct llama_bat
 
         auto detokenized = common_token_to_piece(ctx, batch.token[i]);
 
-        detokenized.erase(
-                std::remove_if(
-                    detokenized.begin(),
-                    detokenized.end(),
-                    [](const unsigned char c) { return !std::isprint(c); }),
-                detokenized.end());
-
         buf << "\n"          << std::to_string(i)
             << ", token '"   << detokenized << "'"
             << ", pos "      << std::to_string(batch.pos[i])