common : add missing declarations

diff --git a/examples/common.cpp b/examples/common.cpp
index bb98d2c66dd2cb5a3bd6ceb6324cdcea932d4c09..e30f524e4058cc64b071f7047aa22fce775f6d35 100644 (file)
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -119,6 +119,10 @@ std::string replace(const std::string & s, const std::string & from, const std::
     return result;
 }
 
+void gpt_vocab::add_special_token(const std::string & token) {
+    special_tokens.push_back(token);
+}
+
 std::map<std::string, int32_t> json_parse(const std::string & fname) {
     std::map<std::string, int32_t> result;
 
@@ -210,10 +214,6 @@ std::map<std::string, int32_t> json_parse(const std::string & fname) {
     return result;
 }
 
-void gpt_vocab::add_special_token(const std::string & token) {
-    special_tokens.push_back(token);
-}
-
 std::string convert_to_utf8(const std::wstring & input) {
     std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
     return converter.to_bytes(input);
@@ -226,7 +226,7 @@ std::wstring convert_to_wstring(const std::string & input) {
 
 std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {
     std::vector<std::string> words;
-    
+
     // first split the text into words
     {
         std::string str = text;

diff --git a/examples/common.h b/examples/common.h
index 73b4a5818091fbeaa7f8b50b5e2a7ad3fe071386..4a24ffbc3f7648a2a328d6ece0d7e11b311576a5 100644 (file)
--- a/examples/common.h
+++ b/examples/common.h
@@ -61,8 +61,9 @@ struct gpt_vocab {
 // poor-man's JSON parsing
 std::map<std::string, int32_t> json_parse(const std::string & fname);
 
-// handle utf-8 coding
-void utf8_to_string(std::string const & in, std::string & out);
+std::string convert_to_utf8(const std::wstring & input);
+
+std::wstring convert_to_wstring(const std::string & input);
 
 // split text into tokens
 //