From: Xuan Son Nguyen <redacted>
Date: Sun, 28 Apr 2024 15:36:18 +0000 (+0200)
Subject: gguf : enforce that tensor names are unique (llama/6905)
X-Git-Tag: upstream/1.7.4~799
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=388c3462a6c38a09e403c4b2b061cd5f154edf52;p=pkg%2Fggml%2Fsources%2Fwhisper.cpp

gguf : enforce that tensor names are unique (llama/6905)

* not allow adding duplicated tensor name

* no duplicated tensor while reading gguf

* typo

* throw exception inside llama_model_loader

Co-authored-by: slaren <redacted>

---------

Co-authored-by: slaren <redacted>
---

diff --git a/ggml.c b/ggml.c
index 7f637543..3bddcdbf 100644
--- a/ggml.c
+++ b/ggml.c
@@ -20890,6 +20890,14 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p
             // TODO: return an error instead of crashing with GGML_ASSERT
             gguf_tensor_info_sanitize(info);
 
+            // make sure there is no duplicated tensor names
+            for (uint64_t j = 0; j < i; ++j) {
+                if (strcmp(info->name.data, ctx->infos[j].name.data) == 0) {
+                    fprintf(stderr, "%s: duplicated tensor name %s\n", __func__, info->name.data);
+                    ok = false;
+                }
+            }
+
             if (!ok) {
                 fprintf(stderr, "%s: failed to read tensor info\n", __func__);
                 fclose(file);
@@ -21426,6 +21434,10 @@ void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src) {
 void gguf_add_tensor(
              struct gguf_context * ctx,
         const struct ggml_tensor * tensor) {
+    if (gguf_find_tensor(ctx, tensor->name) != -1) {
+        GGML_ASSERT(false && "duplicated tensor name");
+    }
+
     const int idx = ctx->header.n_tensors;
     ctx->infos = realloc(ctx->infos, (idx + 1)*sizeof(struct gguf_tensor_info));