From: Kamilake <redacted>
Date: Wed, 22 Mar 2023 19:17:24 +0000 (+0900)
Subject: models : change default encoding to utf8 (#605)
X-Git-Tag: upstream/1.7.4~1545
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=992aa2cd1b328d8beb6ae7bd368f37d128df5647;p=pkg%2Fggml%2Fsources%2Fwhisper.cpp

models : change default encoding to utf8 (#605)
---

diff --git a/models/convert-h5-to-ggml.py b/models/convert-h5-to-ggml.py
index b06ad23d..3ddee220 100644
--- a/models/convert-h5-to-ggml.py
+++ b/models/convert-h5-to-ggml.py
@@ -79,11 +79,11 @@ dir_model   = sys.argv[1]
 dir_whisper = sys.argv[2]
 dir_out     = sys.argv[3]
 
-with open(dir_model + "/vocab.json", "r") as f:
+with open(dir_model + "/vocab.json", "r", encoding="utf8") as f:
     encoder = json.load(f)
-with open(dir_model + "/added_tokens.json", "r") as f:
+with open(dir_model + "/added_tokens.json", "r", encoding="utf8") as f:
     encoder_added = json.load(f)
-with open(dir_model + "/config.json", "r") as f:
+with open(dir_model + "/config.json", "r", encoding="utf8") as f:
     hparams = json.load(f)
 
 model = WhisperForConditionalGeneration.from_pretrained(dir_model)