]> git.djapps.eu Git - pkg/ggml/sources/llama.cpp/commitdiff
convert-hf : Fix the encoding in the convert-hf-to-gguf-update.py (#8040)
authorHamdoud Hakem <redacted>
Thu, 20 Jun 2024 19:59:59 +0000 (20:59 +0100)
committerGitHub <redacted>
Thu, 20 Jun 2024 19:59:59 +0000 (21:59 +0200)
convert-hf-to-gguf-update.py

index fbf1e1ea3de375563d27e68e58b9771ad0cd4335..67598b561e6cb699179c88a1e4f15ac76f5d5e43 100755 (executable)
@@ -214,7 +214,7 @@ src_func = f"""
 """
 
 convert_py_pth = pathlib.Path("convert-hf-to-gguf.py")
-convert_py = convert_py_pth.read_text()
+convert_py = convert_py_pth.read_text(encoding="utf-8")
 convert_py = re.sub(
     r"(# Marker: Start get_vocab_base_pre)(.+?)( +# Marker: End get_vocab_base_pre)",
     lambda m: m.group(1) + src_func + m.group(3),
@@ -222,7 +222,7 @@ convert_py = re.sub(
     flags=re.DOTALL | re.MULTILINE,
 )
 
-convert_py_pth.write_text(convert_py)
+convert_py_pth.write_text(convert_py, encoding="utf-8")
 
 logger.info("+++ convert-hf-to-gguf.py was updated")