model-conversion : add token ids to prompt token output [no ci] (#17863)

This commit adds the token ids to the printed prompt outputs.

The motivation for this is that is can be useful to see the actual token
ids alongside the token strings for debugging.

diff --git a/examples/model-conversion/logits.cpp b/examples/model-conversion/logits.cpp
index bbd095e6034cc358839598fc3c2b1dbf4b06e5f2..5bcf0632677edbe1ac83d6bfc7a0feac3477a054 100644 (file)
--- a/examples/model-conversion/logits.cpp
+++ b/examples/model-conversion/logits.cpp
@@ -144,7 +144,7 @@ int main(int argc, char ** argv) {
             return 1;
         }
         std::string s(buf, n);
-        printf("%s", s.c_str());
+        printf("%s (%d)", s.c_str(), id);
     }
     printf("\n");