From: Caleb DeLeeuw <redacted>
Date: Tue, 4 Aug 2026 07:34:30 +0000 (-0700)
Subject: convert : import bytes_to_unicode from convert_slow_tokenizer (#26217)
X-Git-Tag: upstream/0.0.10438~178
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=b06fbc968b980daa313687152114dd1bfd1edaa9;p=pkg%2Fggml%2Fsources%2Fllama.cpp

convert : import bytes_to_unicode from convert_slow_tokenizer (#26217)

bytes_to_unicode was removed from transformers.models.gpt2.tokenization_gpt2
in huggingface/transformers#40936, but it had already been copied into
transformers.convert_slow_tokenizer in huggingface/transformers#30334
(transformers 4.54.1), so import it directly from there.

Applies the same fix to chatglm.py.
---

diff --git a/conversion/chatglm.py b/conversion/chatglm.py
index 801913075..d63855038 100644
--- a/conversion/chatglm.py
+++ b/conversion/chatglm.py
@@ -81,7 +81,7 @@ class ChatGLMModel(TextModel):
 
     @staticmethod
     def token_bytes_to_string(b):
-        from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode  # ty: ignore[unresolved-import]
+        from transformers.convert_slow_tokenizer import bytes_to_unicode
         byte_encoder = bytes_to_unicode()
         return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])
 
diff --git a/conversion/qwen.py b/conversion/qwen.py
index 7e3d8c0d1..b4ae528bf 100644
--- a/conversion/qwen.py
+++ b/conversion/qwen.py
@@ -18,7 +18,7 @@ class QwenModel(TextModel):
 
     @staticmethod
     def token_bytes_to_string(b):
-        from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode  # ty: ignore[unresolved-import]
+        from transformers.convert_slow_tokenizer import bytes_to_unicode
         byte_encoder = bytes_to_unicode()
         return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])