]> git.djapps.eu Git - pkg/ggml/sources/llama.cpp/commitdiff
py : remove unnecessary hasattr (#4903)
authorGeorgi Gerganov <redacted>
Tue, 16 Jan 2024 18:59:31 +0000 (20:59 +0200)
committerGeorgi Gerganov <redacted>
Tue, 16 Jan 2024 18:59:31 +0000 (20:59 +0200)
convert-hf-to-gguf.py

index b133f3b49f7190c6c180b37a3a81f4bab8726d80..1178d63a231faef29426c7939a10d3aedb30b414 100755 (executable)
@@ -266,11 +266,10 @@ class Model:
                 toktypes.append(gguf.TokenType.USER_DEFINED)
             elif reverse_vocab[i] in added_vocab:
                 tokens.append(reverse_vocab[i])
-                if hasattr(tokenizer, "added_tokens_decoder"):
-                    if tokenizer.added_tokens_decoder[i].special:
-                        toktypes.append(gguf.TokenType.CONTROL)
-                    else:
-                        toktypes.append(gguf.TokenType.USER_DEFINED)
+                if tokenizer.added_tokens_decoder[i].special:
+                    toktypes.append(gguf.TokenType.CONTROL)
+                else:
+                    toktypes.append(gguf.TokenType.USER_DEFINED)
             else:
                 tokens.append(reverse_vocab[i])
                 toktypes.append(gguf.TokenType.NORMAL)