examples : add quantize version to MPT and Replit examples (ref #168)

diff --git a/examples/mpt/main.cpp b/examples/mpt/main.cpp
index d6825310480fb9d821857eedce9c544dfbfefa15..2041d931113a41057d18752b22aff8f9376f675f 100644 (file)
--- a/examples/mpt/main.cpp
+++ b/examples/mpt/main.cpp
@@ -22,14 +22,14 @@ int n_ctx = 4096;
 
 // no defaults for now
 struct mpt_hparams {
-    int32_t d_model = 0;
-    int32_t max_seq_len = 0;
-    int32_t n_heads = 0;
-    int32_t n_layers = 0;
-    int32_t n_vocab = 0;
+    int32_t d_model      = 0;
+    int32_t max_seq_len  = 0;
+    int32_t n_heads      = 0;
+    int32_t n_layers     = 0;
+    int32_t n_vocab      = 0;
     float alibi_bias_max = 0;
-    float clip_qkv = 0;
-    int32_t ftype = 0;
+    float clip_qkv       = 0;
+    int32_t ftype        = 0;
 };
 
 struct mpt_layer {
@@ -88,14 +88,16 @@ bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vo
     {
         auto & hparams = model.hparams;
 
-        fin.read((char *)&hparams.d_model, sizeof(hparams.d_model));
-        fin.read((char *)&hparams.max_seq_len, sizeof(hparams.max_seq_len));
-        fin.read((char *)&hparams.n_heads, sizeof(hparams.n_heads));
-        fin.read((char *)&hparams.n_layers, sizeof(hparams.n_layers));
-        fin.read((char *)&hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *)&hparams.alibi_bias_max, sizeof(hparams.alibi_bias_max));
-        fin.read((char *)&hparams.clip_qkv, sizeof(hparams.clip_qkv));
-        fin.read((char *)&hparams.ftype, sizeof(hparams.ftype));
+        fin.read((char *) &hparams.d_model,        sizeof(hparams.d_model));
+        fin.read((char *) &hparams.max_seq_len,    sizeof(hparams.max_seq_len));
+        fin.read((char *) &hparams.n_heads,        sizeof(hparams.n_heads));
+        fin.read((char *) &hparams.n_layers,       sizeof(hparams.n_layers));
+        fin.read((char *) &hparams.n_vocab,        sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.alibi_bias_max, sizeof(hparams.alibi_bias_max));
+        fin.read((char *) &hparams.clip_qkv,       sizeof(hparams.clip_qkv));
+        fin.read((char *) &hparams.ftype,          sizeof(hparams.ftype));
+
+        const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
 
         printf("%s: d_model        = %d\n", __func__, hparams.d_model);
         printf("%s: max_seq_len    = %d\n", __func__, hparams.max_seq_len);
@@ -105,6 +107,9 @@ bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vo
         printf("%s: alibi_bias_max = %f\n", __func__, hparams.alibi_bias_max);
         printf("%s: clip_qkv       = %f\n", __func__, hparams.clip_qkv);
         printf("%s: ftype          = %d\n", __func__, hparams.ftype);
+        printf("%s: qntvr          = %d\n", __func__, qntvr);
+
+        hparams.ftype %= GGML_QNT_VERSION_FACTOR;
     }
 
     // load vocab

diff --git a/examples/mpt/quantize.cpp b/examples/mpt/quantize.cpp
index 8f32bdd4b77eea821b2f7cf3bbe3dd55c1c7f41f..95b83c36f8fd260feb960937ccfa2d5bf660cb71 100644 (file)
--- a/examples/mpt/quantize.cpp
+++ b/examples/mpt/quantize.cpp
@@ -14,14 +14,14 @@
 #include <vector>
 
 struct mpt_hparams {
-    int32_t d_model = 0;
-    int32_t max_seq_len = 0;
-    int32_t n_heads = 0;
-    int32_t n_layers = 0;
-    int32_t n_vocab = 0;
+    int32_t d_model      = 0;
+    int32_t max_seq_len  = 0;
+    int32_t n_heads      = 0;
+    int32_t n_layers     = 0;
+    int32_t n_vocab      = 0;
     float alibi_bias_max = 0;
-    float clip_qkv = 0;
-    int32_t ftype = 0;
+    float clip_qkv       = 0;
+    int32_t ftype        = 0;
 };
 
 // quantize a model
@@ -61,32 +61,38 @@ bool mpt_model_quantize(const std::string & fname_inp,
 
     // load hparams
     {
-        finp.read((char *)&hparams.d_model, sizeof(hparams.d_model));
-        finp.read((char *)&hparams.max_seq_len, sizeof(hparams.max_seq_len));
-        finp.read((char *)&hparams.n_heads, sizeof(hparams.n_heads));
-        finp.read((char *)&hparams.n_layers, sizeof(hparams.n_layers));
-        finp.read((char *)&hparams.n_vocab, sizeof(hparams.n_vocab));
-        finp.read((char *)&hparams.alibi_bias_max, sizeof(hparams.alibi_bias_max));
-        finp.read((char *)&hparams.clip_qkv, sizeof(hparams.clip_qkv));
-        finp.read((char *)&hparams.ftype, sizeof(hparams.ftype));
-
-        printf("%s: d_model = %d\n", __func__, hparams.d_model);
-        printf("%s: max_seq_len = %d\n", __func__, hparams.max_seq_len);
-        printf("%s: n_heads = %d\n", __func__, hparams.n_heads);
-        printf("%s: n_layers = %d\n", __func__, hparams.n_layers);
-        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
+        finp.read((char *) &hparams.d_model,        sizeof(hparams.d_model));
+        finp.read((char *) &hparams.max_seq_len,    sizeof(hparams.max_seq_len));
+        finp.read((char *) &hparams.n_heads,        sizeof(hparams.n_heads));
+        finp.read((char *) &hparams.n_layers,       sizeof(hparams.n_layers));
+        finp.read((char *) &hparams.n_vocab,        sizeof(hparams.n_vocab));
+        finp.read((char *) &hparams.alibi_bias_max, sizeof(hparams.alibi_bias_max));
+        finp.read((char *) &hparams.clip_qkv,       sizeof(hparams.clip_qkv));
+        finp.read((char *) &hparams.ftype,          sizeof(hparams.ftype));
+
+        const int32_t qntvr_src =    hparams.ftype / GGML_QNT_VERSION_FACTOR;
+        const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;
+
+        printf("%s: d_model        = %d\n", __func__, hparams.d_model);
+        printf("%s: max_seq_len    = %d\n", __func__, hparams.max_seq_len);
+        printf("%s: n_heads        = %d\n", __func__, hparams.n_heads);
+        printf("%s: n_layers       = %d\n", __func__, hparams.n_layers);
+        printf("%s: n_vocab        = %d\n", __func__, hparams.n_vocab);
         printf("%s: alibi_bias_max = %f\n", __func__, hparams.alibi_bias_max);
-        printf("%s: clip_qkv = %f\n", __func__, hparams.clip_qkv);
-        printf("%s: ftype = %d\n", __func__, hparams.ftype);
-
-        fout.write((char *)&hparams.d_model, sizeof(hparams.d_model));
-        fout.write((char *)&hparams.max_seq_len, sizeof(hparams.max_seq_len));
-        fout.write((char *)&hparams.n_heads, sizeof(hparams.n_heads));
-        fout.write((char *)&hparams.n_layers, sizeof(hparams.n_layers));
-        fout.write((char *)&hparams.n_vocab, sizeof(hparams.n_vocab));
-        fout.write((char *)&hparams.alibi_bias_max, sizeof(hparams.alibi_bias_max));
-        fout.write((char *)&hparams.clip_qkv, sizeof(hparams.clip_qkv));
-        fout.write((char *)&ftype, sizeof(hparams.ftype));
+        printf("%s: clip_qkv       = %f\n", __func__, hparams.clip_qkv);
+        printf("%s: ftype (src) = %d\n", __func__, hparams.ftype);
+        printf("%s: qntvr (src) = %d\n", __func__, qntvr_src);
+        printf("%s: ftype (dst) = %d\n", __func__, ftype_dst);
+        printf("%s: qntvr (dst) = %d\n", __func__, GGML_QNT_VERSION);
+
+        fout.write((char *) &hparams.d_model,        sizeof(hparams.d_model));
+        fout.write((char *) &hparams.max_seq_len,    sizeof(hparams.max_seq_len));
+        fout.write((char *) &hparams.n_heads,        sizeof(hparams.n_heads));
+        fout.write((char *) &hparams.n_layers,       sizeof(hparams.n_layers));
+        fout.write((char *) &hparams.n_vocab,        sizeof(hparams.n_vocab));
+        fout.write((char *) &hparams.alibi_bias_max, sizeof(hparams.alibi_bias_max));
+        fout.write((char *) &hparams.clip_qkv,       sizeof(hparams.clip_qkv));
+        fout.write((char *) &ftype_dst,              sizeof(ftype_dst));
     }
 
     // load vocab

diff --git a/examples/replit/main.cpp b/examples/replit/main.cpp
index 2f056c235cecddc480b3404e02d157fb049c11d8..cd678a494b909fadf6ec492d3ba79b93454c9465 100644 (file)
--- a/examples/replit/main.cpp
+++ b/examples/replit/main.cpp
@@ -129,12 +129,12 @@ std::string replit_tokenizer_detokenize(replit_tokenizer & tokenizer, const std:
 
 // no defaults for now
 struct mpt_hparams {
-    int32_t d_model = 0;
+    int32_t d_model     = 0;
     int32_t max_seq_len = 0;
-    int32_t n_heads = 0;
-    int32_t n_layers = 0;
-    int32_t n_vocab = 0;
-    int32_t ftype = 0;
+    int32_t n_heads     = 0;
+    int32_t n_layers    = 0;
+    int32_t n_vocab     = 0;
+    int32_t ftype       = 0;
 };
 
 struct replit_layer {
@@ -195,19 +195,24 @@ bool replit_model_load(const std::string & fname, replit_model & model, replit_t
     {
         auto & hparams = model.hparams;
 
-        fin.read((char *)&hparams.d_model, sizeof(hparams.d_model));
-        fin.read((char *)&hparams.max_seq_len, sizeof(hparams.max_seq_len));
-        fin.read((char *)&hparams.n_heads, sizeof(hparams.n_heads));
-        fin.read((char *)&hparams.n_layers, sizeof(hparams.n_layers));
-        fin.read((char *)&hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *)&hparams.ftype, sizeof(hparams.ftype));
-
-        printf("%s: d_model       = %d\n", __func__, hparams.d_model);
-        printf("%s: max_seq_len   = %d\n", __func__, hparams.max_seq_len);
-        printf("%s: n_heads       = %d\n", __func__, hparams.n_heads);
-        printf("%s: n_layers      = %d\n", __func__, hparams.n_layers);
-        printf("%s: n_vocab      = %d\n", __func__, hparams.n_vocab);
-        printf("%s: ftype   = %d\n", __func__, hparams.ftype);
+        fin.read((char *) &hparams.d_model,     sizeof(hparams.d_model));
+        fin.read((char *) &hparams.max_seq_len, sizeof(hparams.max_seq_len));
+        fin.read((char *) &hparams.n_heads,     sizeof(hparams.n_heads));
+        fin.read((char *) &hparams.n_layers,    sizeof(hparams.n_layers));
+        fin.read((char *) &hparams.n_vocab,     sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.ftype,   sizeof(hparams.ftype));
+
+        const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
+
+        printf("%s: d_model     = %d\n", __func__, hparams.d_model);
+        printf("%s: max_seq_len = %d\n", __func__, hparams.max_seq_len);
+        printf("%s: n_heads     = %d\n", __func__, hparams.n_heads);
+        printf("%s: n_layers    = %d\n", __func__, hparams.n_layers);
+        printf("%s: n_vocab     = %d\n", __func__, hparams.n_vocab);
+        printf("%s: ftype       = %d\n", __func__, hparams.ftype);
+        printf("%s: qntvr       = %d\n", __func__, qntvr);
+
+        hparams.ftype %= GGML_QNT_VERSION_FACTOR;
     }
 
     // load vocab

diff --git a/examples/replit/quantize.cpp b/examples/replit/quantize.cpp
index 40a5806064e7bc5f80896913096fb7b07ef5736d..9a4ec4339bb9191fc0372e0c6372b4a2c9bcd9b1 100644 (file)
--- a/examples/replit/quantize.cpp
+++ b/examples/replit/quantize.cpp
@@ -14,12 +14,12 @@
 #include <vector>
 
 struct mpt_hparams {
-    int32_t d_model = 0;
+    int32_t d_model     = 0;
     int32_t max_seq_len = 0;
-    int32_t n_heads = 0;
-    int32_t n_layers = 0;
-    int32_t n_vocab = 0;
-    int32_t ftype = 0;
+    int32_t n_heads     = 0;
+    int32_t n_layers    = 0;
+    int32_t n_vocab     = 0;
+    int32_t ftype       = 0;
 };
 
 // quantize a model
@@ -59,26 +59,32 @@ bool mpt_model_quantize(const std::string & fname_inp,
 
     // load hparams
     {
-        finp.read((char *)&hparams.d_model, sizeof(hparams.d_model));
-        finp.read((char *)&hparams.max_seq_len, sizeof(hparams.max_seq_len));
-        finp.read((char *)&hparams.n_heads, sizeof(hparams.n_heads));
-        finp.read((char *)&hparams.n_layers, sizeof(hparams.n_layers));
-        finp.read((char *)&hparams.n_vocab, sizeof(hparams.n_vocab));
-        finp.read((char *)&hparams.ftype, sizeof(hparams.ftype));
-
-        printf("%s: d_model = %d\n", __func__, hparams.d_model);
+        finp.read((char *) &hparams.d_model,     sizeof(hparams.d_model));
+        finp.read((char *) &hparams.max_seq_len, sizeof(hparams.max_seq_len));
+        finp.read((char *) &hparams.n_heads,     sizeof(hparams.n_heads));
+        finp.read((char *) &hparams.n_layers,    sizeof(hparams.n_layers));
+        finp.read((char *) &hparams.n_vocab,     sizeof(hparams.n_vocab));
+        finp.read((char *) &hparams.ftype,       sizeof(hparams.ftype));
+
+        const int32_t qntvr_src =    hparams.ftype / GGML_QNT_VERSION_FACTOR;
+        const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;
+
+        printf("%s: d_model     = %d\n", __func__, hparams.d_model);
         printf("%s: max_seq_len = %d\n", __func__, hparams.max_seq_len);
-        printf("%s: n_heads = %d\n", __func__, hparams.n_heads);
-        printf("%s: n_layers = %d\n", __func__, hparams.n_layers);
-        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
-        printf("%s: ftype = %d\n", __func__, hparams.ftype);
-
-        fout.write((char *)&hparams.d_model, sizeof(hparams.d_model));
-        fout.write((char *)&hparams.max_seq_len, sizeof(hparams.max_seq_len));
-        fout.write((char *)&hparams.n_heads, sizeof(hparams.n_heads));
-        fout.write((char *)&hparams.n_layers, sizeof(hparams.n_layers));
-        fout.write((char *)&hparams.n_vocab, sizeof(hparams.n_vocab));
-        fout.write((char *)&ftype, sizeof(hparams.ftype));
+        printf("%s: n_heads     = %d\n", __func__, hparams.n_heads);
+        printf("%s: n_layers    = %d\n", __func__, hparams.n_layers);
+        printf("%s: n_vocab     = %d\n", __func__, hparams.n_vocab);
+        printf("%s: ftype (src) = %d\n", __func__, hparams.ftype);
+        printf("%s: qntvr (src) = %d\n", __func__, qntvr_src);
+        printf("%s: ftype (dst) = %d\n", __func__, ftype_dst);
+        printf("%s: qntvr (dst) = %d\n", __func__, GGML_QNT_VERSION);
+
+        fout.write((char *) &hparams.d_model,     sizeof(hparams.d_model));
+        fout.write((char *) &hparams.max_seq_len, sizeof(hparams.max_seq_len));
+        fout.write((char *) &hparams.n_heads,     sizeof(hparams.n_heads));
+        fout.write((char *) &hparams.n_layers,    sizeof(hparams.n_layers));
+        fout.write((char *) &hparams.n_vocab,     sizeof(hparams.n_vocab));
+        fout.write((char *) &ftype_dst,           sizeof(ftype_dst));
     }
 
     // load vocab