From: Tim Miller <redacted>
Date: Sat, 26 Aug 2023 18:27:07 +0000 (+0900)
Subject: llama : use Unicode Escape Sequence to replace encoded characters (#2814)
X-Git-Tag: gguf-v0.4.0~225
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=c7d92e6dfec3f54849f3a0ba373054d29f321ea2;p=pkg%2Fggml%2Fsources%2Fllama.cpp

llama : use Unicode Escape Sequence to replace encoded characters (#2814)

The use of special characters within source files can break compiling on some computers with different region and language settings. Using Unicode escape sequences should allow for the code to be compiled on all setups without needing to change your computers settings or switch regions.
---

diff --git a/llama.cpp b/llama.cpp
index 62889b3e..05c54c21 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -955,10 +955,10 @@ struct llama_vocab {
     id linefeed_id = 13;
 
     int find_bpe_rank(std::string token_left, std::string token_right) const {
-        replace_all(token_left,  " ",  "Ä ");
-        replace_all(token_left,  "\n", "Ċ");
-        replace_all(token_right, " ",  "Ä ");
-        replace_all(token_right, "\n", "Ċ");
+        replace_all(token_left,  " ",  "\u0120");
+        replace_all(token_left,  "\n", "\u010A");
+        replace_all(token_right, " ",  "\u0120");
+        replace_all(token_right, "\n", "\u010A");
 
         auto it = bpe_ranks.find(std::make_pair(token_left, token_right));
         if (it == bpe_ranks.end()) {