From: Georgi Gerganov <redacted>
Date: Mon, 6 Jan 2025 13:36:08 +0000 (+0200)
Subject: server : fix extra BOS in infill endpoint (#11106)
X-Git-Tag: upstream/0.0.4488~60
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=e6e7c75d94adf4d39e846d30807c531ff22865e7;p=pkg%2Fggml%2Fsources%2Fllama.cpp

server : fix extra BOS in infill endpoint (#11106)

* server : fix extra BOS in infill endpoing

ggml-ci

* server : update infill tests
---

diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index c2e62ba6..127323e7 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -3797,7 +3797,7 @@ int main(int argc, char ** argv) {
         data["input_extra"] = input_extra; // default to empty array if it's not exist
 
         std::string prompt = json_value(data, "prompt", std::string());
-        std::vector<llama_tokens> tokenized_prompts = tokenize_input_prompts(ctx_server.ctx, prompt, true, true);
+        std::vector<llama_tokens> tokenized_prompts = tokenize_input_prompts(ctx_server.ctx, prompt, false, true);
         SRV_DBG("creating infill tasks, n_prompts = %d\n", (int) tokenized_prompts.size());
         data["prompt"] = format_infill(
             ctx_server.ctx,
diff --git a/examples/server/tests/unit/test_infill.py b/examples/server/tests/unit/test_infill.py
index ad4b8192..10554db0 100644
--- a/examples/server/tests/unit/test_infill.py
+++ b/examples/server/tests/unit/test_infill.py
@@ -18,7 +18,7 @@ def test_infill_without_input_extra():
         "input_suffix": "}\n",
     })
     assert res.status_code == 200
-    assert match_regex("(Ann|small|shiny)+", res.body["content"])
+    assert match_regex("(Ann|small|shiny|Daddy)+", res.body["content"])
 
 
 def test_infill_with_input_extra():