From: Georgi Gerganov <redacted>
Date: Mon, 12 Aug 2024 07:21:50 +0000 (+0300)
Subject: server : handle models with missing EOS token (#8997)
X-Git-Tag: upstream/0.0.4488~916
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=5ef07e25ac39e62297a67208c5bcced50835a2dd;p=pkg%2Fggml%2Fsources%2Fllama.cpp

server : handle models with missing EOS token (#8997)

ggml-ci
---

diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index 360f571e..1621c7c4 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -631,6 +631,7 @@ struct server_context {
 
     bool clean_kv_cache = true;
     bool add_bos_token  = true;
+    bool has_eos_token  = false;
 
     int32_t n_ctx; // total context for all clients / slots
 
@@ -693,7 +694,7 @@ struct server_context {
         n_ctx = llama_n_ctx(ctx);
 
         add_bos_token = llama_should_add_bos_token(model);
-        GGML_ASSERT(llama_add_eos_token(model) != 1);
+        has_eos_token = llama_add_eos_token(model) != 1;
 
         return true;
     }
@@ -1031,7 +1032,7 @@ struct server_context {
         {
             slot.sparams.logit_bias.clear();
 
-            if (json_value(data, "ignore_eos", false)) {
+            if (json_value(data, "ignore_eos", false) && has_eos_token) {
                 slot.sparams.logit_bias[llama_token_eos(model)] = -INFINITY;
             }