]> git.djapps.eu Git - pkg/ggml/sources/llama.cpp/commitdiff
convert : fix TypeError on GPT-2 vocab.json (#5288)
authorSang-Kil Park <redacted>
Wed, 7 Feb 2024 04:28:00 +0000 (13:28 +0900)
committerGitHub <redacted>
Wed, 7 Feb 2024 04:28:00 +0000 (23:28 -0500)
convert.py

index 4a2847a274d59fea75ad4652337fe33cfee71169..323e8058d5ee95ee49cd625093bd0ecd3ea28525 100755 (executable)
@@ -334,9 +334,9 @@ class Params:
 class BpeVocab:
     def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None:
         self.bpe_tokenizer = json.loads(open(str(fname_tokenizer), encoding="utf-8").read())
-        try:
+        if isinstance(self.bpe_tokenizer.get('model'), dict):
             self.vocab = self.bpe_tokenizer["model"]["vocab"]
-        except KeyError:
+        else:
             self.vocab = self.bpe_tokenizer
         added_tokens: dict[str, int]
         if fname_added_tokens is not None: