Fix for #3454 (#3455)

Fix: `sentencepiece` tokenizers with added tokens failed with an incorrect assertion

diff --git a/llama.cpp b/llama.cpp
index 79ea2b235602eb167be70a61e78bca62a7a592f2..89dc403b908474cab452679cccc470ed9b2e9aa9 100644 (file)
--- a/llama.cpp
+++ b/llama.cpp
@@ -8200,7 +8200,9 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, ch
                 buf[0] = llama_token_to_byte(model->vocab, token);
                 return 1;
             } else {
-                GGML_ASSERT(false);
+                // TODO: for now we accept all unsupported token types,
+                // suppressing them like CONTROL tokens.
+                // GGML_ASSERT(false);
             }
             break;
         }
@@ -8216,7 +8218,9 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, ch
             } else if (llama_is_control_token(model->vocab, token)) {
                 ;
             } else {
-                GGML_ASSERT(false);
+                // TODO: for now we accept all unsupported token types,
+                // suppressing them like CONTROL tokens.
+                // GGML_ASSERT(false);
             }
             break;
         }