gpt : add support for gpt-jt + fix unicode support

diff --git a/examples/gpt-2/convert-ckpt-to-ggml.py b/examples/gpt-2/convert-ckpt-to-ggml.py
index 09824a804cb81a1ff9864b70ff22e42f7be7d8fc..7ae438013a08341431a67af78c76526f9408f952 100644 (file)
--- a/examples/gpt-2/convert-ckpt-to-ggml.py
+++ b/examples/gpt-2/convert-ckpt-to-ggml.py
@@ -81,8 +81,9 @@ byte_encoder = bytes_to_unicode()
 byte_decoder = {v:k for k, v in byte_encoder.items()}
 
 fout.write(struct.pack("i", len(encoder)))
+
 for key in encoder:
-    text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8')
+    text = bytearray([byte_decoder[c] for c in key])
     fout.write(struct.pack("i", len(text)))
     fout.write(text)
 
@@ -105,6 +106,10 @@ for name, shape in list_vars:
             print("  Converting to float16")
             data = data.astype(np.float16)
             ftype = 1
+        else:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype = 0
 
     # for efficiency - transpose the projection matrices
     if name[-13:] == "/mlp/c_proj/w":

diff --git a/examples/gpt-j/convert-h5-to-ggml.py b/examples/gpt-j/convert-h5-to-ggml.py
index a1efecba09ba4a897a64d96fc5de8d71f09a9312..310e60e0a06a269bbd6f43255ee75a4031a01452 100644 (file)
--- a/examples/gpt-j/convert-h5-to-ggml.py
+++ b/examples/gpt-j/convert-h5-to-ggml.py
@@ -91,13 +91,14 @@ byte_encoder = bytes_to_unicode()
 byte_decoder = {v:k for k, v in byte_encoder.items()}
 
 fout.write(struct.pack("i", len(encoder) + len(encoder_added)))
+
 for key in encoder:
-    text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8')
+    text = bytearray([byte_decoder[c] for c in key])
     fout.write(struct.pack("i", len(text)))
     fout.write(text)
 
 for key in encoder_added:
-    text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8')
+    text = bytearray([byte_decoder[c] for c in key])
     fout.write(struct.pack("i", len(text)))
     fout.write(text)
 
@@ -119,6 +120,10 @@ for name in list_vars.keys():
             print("  Converting to float16")
             data = data.astype(np.float16)
             ftype = 1
+        else:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype = 0
 
     # for efficiency - transpose these matrices:
     #  "transformer.h.*.mlp.fc_in.weight