From: Alexey Dubrov <redacted>
Date: Tue, 3 Feb 2026 06:31:01 +0000 (+0300)
Subject: vocab: add Falcon-H1-Tiny-Coder FIM tokens (#19249)
X-Git-Tag: upstream/0.0.8067~148
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=1efb5f7ae120c7cc7a33c4d1d82a05b3c50122f6;p=pkg%2Fggml%2Fsources%2Fllama.cpp

vocab: add Falcon-H1-Tiny-Coder FIM tokens (#19249)
---

diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 74a8496f9..38d03a8c3 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -2262,6 +2262,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                         || t.first == "<PRE>"
                         || t.first == "▁<PRE>"          // CodeLlama
                         || t.first == "<|code_prefix|>" // GLM-4.5
+                        || t.first == "<|prefix|>"      // Falcon-H1-Tiny-Coder
                         ) {
                     special_fim_pre_id = t.second;
                     if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
@@ -2282,6 +2283,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                         || t.first == "<SUF>"
                         || t.first == "▁<SUF>"         // CodeLlama
                         || t.first == "<|code_suffix|>" // GLM-4.5
+                        || t.first == "<|suffix|>"      // Falcon-H1-Tiny-Coder
                         ) {
                     special_fim_suf_id = t.second;
                     if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
@@ -2302,6 +2304,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                         || t.first == "<MID>"
                         || t.first == "▁<MID>"         // CodeLlama
                         || t.first == "<|code_middle|>" // GLM-4.5
+                        || t.first == "<|middle|>"      // Falcon-H1-Tiny-Coder
                         ) {
                     special_fim_mid_id = t.second;
                     if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {