From: Georgi Gerganov <redacted>
Date: Sun, 4 Dec 2022 16:33:14 +0000 (+0200)
Subject: gpt : add support for gpt-jt + fix unicode support
X-Git-Tag: upstream/0.0.1642~1603
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=ed09c7190ea26f68faf9adba57feb3c7f404a26d;p=pkg%2Fggml%2Fsources%2Fggml

gpt : add support for gpt-jt + fix unicode support
---

diff --git a/examples/gpt-2/convert-ckpt-to-ggml.py b/examples/gpt-2/convert-ckpt-to-ggml.py
index 09824a80..7ae43801 100644
--- a/examples/gpt-2/convert-ckpt-to-ggml.py
+++ b/examples/gpt-2/convert-ckpt-to-ggml.py
@@ -81,8 +81,9 @@ byte_encoder = bytes_to_unicode()
 byte_decoder = {v:k for k, v in byte_encoder.items()}
 
 fout.write(struct.pack("i", len(encoder)))
+
 for key in encoder:
-    text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8')
+    text = bytearray([byte_decoder[c] for c in key])
     fout.write(struct.pack("i", len(text)))
     fout.write(text)
 
@@ -105,6 +106,10 @@ for name, shape in list_vars:
             print("  Converting to float16")
             data = data.astype(np.float16)
             ftype = 1
+        else:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype = 0
 
     # for efficiency - transpose the projection matrices
     if name[-13:] == "/mlp/c_proj/w":
diff --git a/examples/gpt-j/convert-h5-to-ggml.py b/examples/gpt-j/convert-h5-to-ggml.py
index a1efecba..310e60e0 100644
--- a/examples/gpt-j/convert-h5-to-ggml.py
+++ b/examples/gpt-j/convert-h5-to-ggml.py
@@ -91,13 +91,14 @@ byte_encoder = bytes_to_unicode()
 byte_decoder = {v:k for k, v in byte_encoder.items()}
 
 fout.write(struct.pack("i", len(encoder) + len(encoder_added)))
+
 for key in encoder:
-    text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8')
+    text = bytearray([byte_decoder[c] for c in key])
     fout.write(struct.pack("i", len(text)))
     fout.write(text)
 
 for key in encoder_added:
-    text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8')
+    text = bytearray([byte_decoder[c] for c in key])
     fout.write(struct.pack("i", len(text)))
     fout.write(text)
 
@@ -119,6 +120,10 @@ for name in list_vars.keys():
             print("  Converting to float16")
             data = data.astype(np.float16)
             ftype = 1
+        else:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype = 0
 
     # for efficiency - transpose these matrices:
     #  "transformer.h.*.mlp.fc_in.weight