From: Georgi Gerganov Date: Sat, 20 May 2023 14:33:07 +0000 (+0300) Subject: examples : fix vocab loading (close #163) X-Git-Tag: upstream/0.0.1642~1457 X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=8f9fc0af4cbc0df77b6147bc6ce93dfb0931be3b;p=pkg%2Fggml%2Fsources%2Fggml examples : fix vocab loading (close #163) --- diff --git a/examples/dolly-v2/main.cpp b/examples/dolly-v2/main.cpp index 235e35f2..e36d6e0e 100644 --- a/examples/dolly-v2/main.cpp +++ b/examples/dolly-v2/main.cpp @@ -114,7 +114,7 @@ bool dollyv2_model_load(const std::string & fname, dollyv2_model & model, gpt_vo fin.read((char *) &hparams.n_head, sizeof(hparams.n_head)); fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer)); fin.read((char *) &hparams.n_rot, sizeof(hparams.n_rot)); - fin.read((char *) &hparams.par_res, sizeof(hparams.par_res)); + fin.read((char *) &hparams.par_res, sizeof(hparams.par_res)); fin.read((char *) &hparams.ftype, sizeof(hparams.ftype)); const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR; @@ -137,12 +137,15 @@ bool dollyv2_model_load(const std::string & fname, dollyv2_model & model, gpt_vo const int32_t n_vocab = model.hparams.n_vocab; std::string word; + std::vector buf(128); + for (int i = 0; i < n_vocab; i++) { uint32_t len; fin.read((char *) &len, sizeof(len)); - word.resize(len); - fin.read((char *) word.data(), len); + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); vocab.token_to_id[word] = i; vocab.id_to_token[i] = word; @@ -613,7 +616,7 @@ bool dollyv2_eval( // input for next layer inpL = ggml_add(ctx0, cur, inpL); } - + } // norm diff --git a/examples/gpt-2/main.cpp b/examples/gpt-2/main.cpp index 7d83b710..5d227934 100644 --- a/examples/gpt-2/main.cpp +++ b/examples/gpt-2/main.cpp @@ -125,12 +125,15 @@ bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & } std::string word; + std::vector buf(128); + for (int i = 0; i < n_vocab; i++) { uint32_t len; fin.read((char *) &len, sizeof(len)); - word.resize(len); - fin.read((char *) word.data(), len); + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); vocab.token_to_id[word] = i; vocab.id_to_token[i] = word; diff --git a/examples/gpt-j/main.cpp b/examples/gpt-j/main.cpp index 860d6379..8fd9362f 100644 --- a/examples/gpt-j/main.cpp +++ b/examples/gpt-j/main.cpp @@ -126,12 +126,15 @@ bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & } std::string word; + std::vector buf(128); + for (int i = 0; i < n_vocab; i++) { uint32_t len; fin.read((char *) &len, sizeof(len)); - word.resize(len); - fin.read((char *) word.data(), len); + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); vocab.token_to_id[word] = i; vocab.id_to_token[i] = word; diff --git a/examples/gpt-neox/main.cpp b/examples/gpt-neox/main.cpp index 52f4b060..dcb6ccf1 100644 --- a/examples/gpt-neox/main.cpp +++ b/examples/gpt-neox/main.cpp @@ -127,12 +127,15 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt_ const int32_t n_vocab = model.hparams.n_vocab; std::string word; + std::vector buf(128); + for (int i = 0; i < n_vocab; i++) { uint32_t len; fin.read((char *) &len, sizeof(len)); - word.resize(len); - fin.read((char *) word.data(), len); + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); vocab.token_to_id[word] = i; vocab.id_to_token[i] = word; diff --git a/examples/mpt/main.cpp b/examples/mpt/main.cpp index 44da3830..d6825310 100644 --- a/examples/mpt/main.cpp +++ b/examples/mpt/main.cpp @@ -109,15 +109,18 @@ bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vo // load vocab { - int32_t n_vocab = model.hparams.n_vocab; + const int32_t n_vocab = model.hparams.n_vocab; std::string word; + std::vector buf(128); + for (int i = 0; i < n_vocab; i++) { uint32_t len; - fin.read((char *)&len, sizeof(len)); + fin.read((char *) &len, sizeof(len)); - word.resize(len); - fin.read((char *)word.data(), len); + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); vocab.token_to_id[word] = i; vocab.id_to_token[i] = word; diff --git a/examples/replit/main.cpp b/examples/replit/main.cpp index 96e13559..2f056c23 100644 --- a/examples/replit/main.cpp +++ b/examples/replit/main.cpp @@ -71,15 +71,16 @@ std::pair, float> encode_word(const std::string & word, } bool replit_tokenizer_load(replit_tokenizer & tokenizer, std::istream & fin, int max_vocab_size) { + std::string word; + std::vector buf(128); for (std::size_t i = 0; i < max_vocab_size; i++) { - uint32_t len; fin.read((char *)&len, sizeof(len)); - std::string word; - word.resize(len); - fin.read((char *)word.data(), len); + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); float score; fin.read((char *)&score, sizeof(score)); diff --git a/examples/starcoder/main.cpp b/examples/starcoder/main.cpp index e43f9923..3d8a4df1 100644 --- a/examples/starcoder/main.cpp +++ b/examples/starcoder/main.cpp @@ -126,12 +126,15 @@ bool starcoder_model_load(const std::string & fname, starcoder_model & model, gp } std::string word; + std::vector buf(128); + for (int i = 0; i < n_vocab; i++) { uint32_t len; fin.read((char *) &len, sizeof(len)); - word.resize(len); - fin.read((char *) word.data(), len); + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); vocab.token_to_id[word] = i; vocab.id_to_token[i] = word;