From: Georgi Gerganov <redacted>
Date: Wed, 8 May 2024 09:47:07 +0000 (+0300)
Subject: py : also print the normalizers
X-Git-Tag: upstream/0.0.4488~1677
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=7e0b6a7b3ba94ff624dc27c1e0e735fded8819b8;p=pkg%2Fggml%2Fsources%2Fllama.cpp

py : also print the normalizers
---

diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py
index ae901e24..a3fe67ee 100755
--- a/convert-hf-to-gguf-update.py
+++ b/convert-hf-to-gguf-update.py
@@ -151,6 +151,8 @@ for model in models:
     # print the "pre_tokenizer" content from the tokenizer.json
     with open(f"models/tokenizers/{name}/tokenizer.json", "r", encoding="utf-8") as f:
         cfg = json.load(f)
+        normalizer = cfg["normalizer"]
+        logger.info("normalizer: " + json.dumps(normalizer, indent=4))
         pre_tokenizer = cfg["pre_tokenizer"]
         logger.info("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))