tokenize example: Respect normal add BOS token behavior (#4126)

Allow building with Makefile

diff --git a/Makefile b/Makefile
index 7150dbaf602947391331fb2f7e85f92e1ce3e3f6..a6d2c2ec0f380e5f589de7f91319457543243078 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@
 BUILD_TARGETS = \
        main quantize quantize-stats perplexity embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \
        simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama beam-search  \
-       speculative infill benchmark-matmult parallel finetune export-lora tests/test-c.o
+       speculative infill tokenize benchmark-matmult parallel finetune export-lora tests/test-c.o
 
 # Binaries only useful for tests
 TEST_TARGETS = \
@@ -594,6 +594,9 @@ infill: examples/infill/infill.cpp                            ggml.o llama.o $(C
 simple: examples/simple/simple.cpp                            ggml.o llama.o $(COMMON_DEPS) $(OBJS)
        $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
 
+tokenize: examples/tokenize/tokenize.cpp                      ggml.o llama.o $(COMMON_DEPS) $(OBJS)
+       $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
+
 batched: examples/batched/batched.cpp                         ggml.o llama.o $(COMMON_DEPS) $(OBJS)
        $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
 

diff --git a/examples/tokenize/tokenize.cpp b/examples/tokenize/tokenize.cpp
index 17166836ac127e2bb508c824a6d49b6c9736470f..4ff8e3fa727493c18144e52e8ac06ebd17b19d4a 100644 (file)
--- a/examples/tokenize/tokenize.cpp
+++ b/examples/tokenize/tokenize.cpp
@@ -26,7 +26,7 @@ int main(int argc, char ** argv) {
     llama_context_params ctx_params = llama_context_default_params();
     llama_context * ctx = llama_new_context_with_model(model, ctx_params);
 
-    const bool add_bos = true;
+    const bool add_bos = llama_should_add_bos_token(model);
 
     std::vector<llama_token> tokens;