From: Georgi Gerganov <redacted>
Date: Mon, 22 Apr 2024 12:41:11 +0000 (+0300)
Subject: llama : fix typo in <|im_end|> token text (#6745)
X-Git-Tag: upstream/0.0.4488~1774
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=8960fe86ae075c846c5df8848230d1904ba8877f;p=pkg%2Fggml%2Fsources%2Fllama.cpp

llama : fix typo in <|im_end|> token text (#6745)
---

diff --git a/llama.cpp b/llama.cpp
index 7440c740..a25d115c 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -4340,7 +4340,7 @@ static void llm_load_vocab(
             }
         }
 
-        // find EOT token: "<|eot_id|>", "<|im_emd|>", "<end_of_turn>", etc.
+        // find EOT token: "<|eot_id|>", "<|im_end|>", "<end_of_turn>", etc.
         //
         // TODO: convert scripts should provide this token through the KV metadata LLAMA_KV_TOKENIZER_EOT_ID
         //       for now, we apply this workaround to find the EOT token based on its text
@@ -4351,7 +4351,7 @@ static void llm_load_vocab(
                         //       need to fix convert script
                         //vocab.id_to_token[t.second].type == LLAMA_TOKEN_TYPE_CONTROL &&
                         (t.first == "<|eot_id|>" ||
-                         t.first == "<|im_emd|>" ||
+                         t.first == "<|im_end|>" ||
                          t.first == "<end_of_turn>"
                         )
                    ) {