From: jaime-m-p <redacted>
Date: Tue, 4 Jun 2024 23:26:14 +0000 (+0200)
Subject: Fix per token atrributes bits (#7749)
X-Git-Tag: upstream/0.0.4488~1398
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=c90dbe026b456a233f8f0fbe752212e6a0503ca2;p=pkg%2Fggml%2Fsources%2Fllama.cpp

Fix per token atrributes bits (#7749)
---

diff --git a/llama.h b/llama.h
index b2a302da..9dcd67be 100644
--- a/llama.h
+++ b/llama.h
@@ -109,16 +109,16 @@ extern "C" {
 
     enum llama_token_attr {
         LLAMA_TOKEN_ATTR_UNDEFINED    = 0,
-        LLAMA_TOKEN_ATTR_UNKNOWN      = 1 <<  1,
-        LLAMA_TOKEN_ATTR_UNUSED       = 1 <<  2,
-        LLAMA_TOKEN_ATTR_NORMAL       = 1 <<  3,
-        LLAMA_TOKEN_ATTR_CONTROL      = 1 <<  4,  // SPECIAL?
-        LLAMA_TOKEN_ATTR_USER_DEFINED = 1 <<  5,
-        LLAMA_TOKEN_ATTR_BYTE         = 1 <<  6,
-        LLAMA_TOKEN_ATTR_NORMALIZED   = 1 <<  7,
-        LLAMA_TOKEN_ATTR_LSTRIP       = 1 <<  8,
-        LLAMA_TOKEN_ATTR_RSTRIP       = 1 <<  9,
-        LLAMA_TOKEN_ATTR_SINGLE_WORD  = 1 << 10,
+        LLAMA_TOKEN_ATTR_UNKNOWN      = 1 << 0,
+        LLAMA_TOKEN_ATTR_UNUSED       = 1 << 1,
+        LLAMA_TOKEN_ATTR_NORMAL       = 1 << 2,
+        LLAMA_TOKEN_ATTR_CONTROL      = 1 << 3,  // SPECIAL?
+        LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,
+        LLAMA_TOKEN_ATTR_BYTE         = 1 << 5,
+        LLAMA_TOKEN_ATTR_NORMALIZED   = 1 << 6,
+        LLAMA_TOKEN_ATTR_LSTRIP       = 1 << 7,
+        LLAMA_TOKEN_ATTR_RSTRIP       = 1 << 8,
+        LLAMA_TOKEN_ATTR_SINGLE_WORD  = 1 << 9,
     };
 
     // model file types