examples : fix vocab loading (close #163)

diff --git a/examples/dolly-v2/main.cpp b/examples/dolly-v2/main.cpp
index 235e35f27b9d52e8fcd50ce8e3fb0652c3c93c36..e36d6e0ea9ca9c9347566e5839563bd111622ad0 100644 (file)
--- a/examples/dolly-v2/main.cpp
+++ b/examples/dolly-v2/main.cpp
@@ -114,7 +114,7 @@ bool dollyv2_model_load(const std::string & fname, dollyv2_model & model, gpt_vo
         fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
         fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
         fin.read((char *) &hparams.n_rot,   sizeof(hparams.n_rot));
-        fin.read((char *) &hparams.par_res, sizeof(hparams.par_res));        
+        fin.read((char *) &hparams.par_res, sizeof(hparams.par_res));
         fin.read((char *) &hparams.ftype,   sizeof(hparams.ftype));
 
         const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
@@ -137,12 +137,15 @@ bool dollyv2_model_load(const std::string & fname, dollyv2_model & model, gpt_vo
         const int32_t n_vocab = model.hparams.n_vocab;
 
         std::string word;
+        std::vector<char> buf(128);
+
         for (int i = 0; i < n_vocab; i++) {
             uint32_t len;
             fin.read((char *) &len, sizeof(len));
 
-            word.resize(len);
-            fin.read((char *) word.data(), len);
+            buf.resize(len);
+            fin.read((char *) buf.data(), len);
+            word.assign(buf.data(), len);
 
             vocab.token_to_id[word] = i;
             vocab.id_to_token[i] = word;
@@ -613,7 +616,7 @@ bool dollyv2_eval(
             // input for next layer
             inpL = ggml_add(ctx0, cur, inpL);
         }
-    
+
     }
 
     // norm

diff --git a/examples/gpt-2/main.cpp b/examples/gpt-2/main.cpp
index 7d83b710ef8dcf23ced203df8dbe0a9ced21a811..5d22793464f6722a1b848aeb925079c001b684d4 100644 (file)
--- a/examples/gpt-2/main.cpp
+++ b/examples/gpt-2/main.cpp
@@ -125,12 +125,15 @@ bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab &
         }
 
         std::string word;
+        std::vector<char> buf(128);
+
         for (int i = 0; i < n_vocab; i++) {
             uint32_t len;
             fin.read((char *) &len, sizeof(len));
 
-            word.resize(len);
-            fin.read((char *) word.data(), len);
+            buf.resize(len);
+            fin.read((char *) buf.data(), len);
+            word.assign(buf.data(), len);
 
             vocab.token_to_id[word] = i;
             vocab.id_to_token[i] = word;

diff --git a/examples/gpt-j/main.cpp b/examples/gpt-j/main.cpp
index 860d63798c2b2e00fa2530a929bfc3c7264ead30..8fd9362ff5134647135b1300049e2277016e6c69 100644 (file)
--- a/examples/gpt-j/main.cpp
+++ b/examples/gpt-j/main.cpp
@@ -126,12 +126,15 @@ bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab &
         }
 
         std::string word;
+        std::vector<char> buf(128);
+
         for (int i = 0; i < n_vocab; i++) {
             uint32_t len;
             fin.read((char *) &len, sizeof(len));
 
-            word.resize(len);
-            fin.read((char *) word.data(), len);
+            buf.resize(len);
+            fin.read((char *) buf.data(), len);
+            word.assign(buf.data(), len);
 
             vocab.token_to_id[word] = i;
             vocab.id_to_token[i] = word;

diff --git a/examples/gpt-neox/main.cpp b/examples/gpt-neox/main.cpp
index 52f4b0609c52835109ec14a596a145975f10be6b..dcb6ccf16ff58cab2b594fc6d0d125d473e2f83c 100644 (file)
--- a/examples/gpt-neox/main.cpp
+++ b/examples/gpt-neox/main.cpp
@@ -127,12 +127,15 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt_
         const int32_t n_vocab = model.hparams.n_vocab;
 
         std::string word;
+        std::vector<char> buf(128);
+
         for (int i = 0; i < n_vocab; i++) {
             uint32_t len;
             fin.read((char *) &len, sizeof(len));
 
-            word.resize(len);
-            fin.read((char *) word.data(), len);
+            buf.resize(len);
+            fin.read((char *) buf.data(), len);
+            word.assign(buf.data(), len);
 
             vocab.token_to_id[word] = i;
             vocab.id_to_token[i] = word;

diff --git a/examples/mpt/main.cpp b/examples/mpt/main.cpp
index 44da38300a94f107cad88ba451dffe64c35bf6db..d6825310480fb9d821857eedce9c544dfbfefa15 100644 (file)
--- a/examples/mpt/main.cpp
+++ b/examples/mpt/main.cpp
@@ -109,15 +109,18 @@ bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vo
 
     // load vocab
     {
-        int32_t n_vocab = model.hparams.n_vocab;
+        const int32_t n_vocab = model.hparams.n_vocab;
 
         std::string word;
+        std::vector<char> buf(128);
+
         for (int i = 0; i < n_vocab; i++) {
             uint32_t len;
-            fin.read((char *)&len, sizeof(len));
+            fin.read((char *) &len, sizeof(len));
 
-            word.resize(len);
-            fin.read((char *)word.data(), len);
+            buf.resize(len);
+            fin.read((char *) buf.data(), len);
+            word.assign(buf.data(), len);
 
             vocab.token_to_id[word] = i;
             vocab.id_to_token[i] = word;

diff --git a/examples/replit/main.cpp b/examples/replit/main.cpp
index 96e13559d7f34807509f97f014c9ed41a76db0d2..2f056c235cecddc480b3404e02d157fb049c11d8 100644 (file)
--- a/examples/replit/main.cpp
+++ b/examples/replit/main.cpp
@@ -71,15 +71,16 @@ std::pair<std::vector<std::size_t>, float> encode_word(const std::string & word,
 }
 
 bool replit_tokenizer_load(replit_tokenizer & tokenizer, std::istream & fin, int max_vocab_size) {
+    std::string word;
+    std::vector<char> buf(128);
 
     for (std::size_t i = 0; i < max_vocab_size; i++) {
-
         uint32_t len;
         fin.read((char *)&len, sizeof(len));
 
-        std::string word;
-        word.resize(len);
-        fin.read((char *)word.data(), len);
+        buf.resize(len);
+        fin.read((char *) buf.data(), len);
+        word.assign(buf.data(), len);
 
         float score;
         fin.read((char *)&score, sizeof(score));

diff --git a/examples/starcoder/main.cpp b/examples/starcoder/main.cpp
index e43f9923634c2adafaf8f7dd400f9943ee0e2752..3d8a4df16d9c148c132460fb896223f2802e19e6 100644 (file)
--- a/examples/starcoder/main.cpp
+++ b/examples/starcoder/main.cpp
@@ -126,12 +126,15 @@ bool starcoder_model_load(const std::string & fname, starcoder_model & model, gp
         }
 
         std::string word;
+        std::vector<char> buf(128);
+
         for (int i = 0; i < n_vocab; i++) {
             uint32_t len;
             fin.read((char *) &len, sizeof(len));
 
-            word.resize(len);
-            fin.read((char *) word.data(), len);
+            buf.resize(len);
+            fin.read((char *) buf.data(), len);
+            word.assign(buf.data(), len);
 
             vocab.token_to_id[word] = i;
             vocab.id_to_token[i] = word;