starcoder : add special tokens for fill-in-the-middle task (#269)

diff --git a/examples/starcoder/main.cpp b/examples/starcoder/main.cpp
index 22be994e720ed6e40ba0c64f2bee9b84a7ba2400..0d05d5e25f94fd374975d32a61f777d8b54dae1e 100644 (file)
--- a/examples/starcoder/main.cpp
+++ b/examples/starcoder/main.cpp
@@ -146,6 +146,10 @@ bool starcoder_model_load(const std::string & fname, starcoder_model & model, gp
                 "<|user|>",
                 "<|assistant|>",
                 "<|end|>",
+                "<fim-prefix>",
+                "<fim-middle>",
+                "<fim-suffix>",
+                "<fim-pad>"
             }) {
             if (vocab.token_to_id.find(token) != vocab.token_to_id.end()) {
                 vocab.add_special_token(token);