From: AmbientL <redacted>
Date: Sat, 24 Jun 2023 15:30:23 +0000 (+0000)
Subject: starcoder : add special tokens for fill-in-the-middle task (#269)
X-Git-Tag: upstream/0.0.1642~1396
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=aa31b0bf591ee84a99ee674dba0aceddfe8e37ea;p=pkg%2Fggml%2Fsources%2Fggml

starcoder : add special tokens for fill-in-the-middle task (#269)
---

diff --git a/examples/starcoder/main.cpp b/examples/starcoder/main.cpp
index 22be994e..0d05d5e2 100644
--- a/examples/starcoder/main.cpp
+++ b/examples/starcoder/main.cpp
@@ -146,6 +146,10 @@ bool starcoder_model_load(const std::string & fname, starcoder_model & model, gp
                 "<|user|>",
                 "<|assistant|>",
                 "<|end|>",
+                "<fim-prefix>",
+                "<fim-middle>",
+                "<fim-suffix>",
+                "<fim-pad>"
             }) {
             if (vocab.token_to_id.find(token) != vocab.token_to_id.end()) {
                 vocab.add_special_token(token);