{ LLM_ARCH_TALKIE, "talkie" },
{ LLM_ARCH_MELLUM, "mellum" },
{ LLM_ARCH_NANBEIGE, "nanbeige" },
+ { LLM_ARCH_QWEN3TTS, "qwen3tts" },
{ LLM_ARCH_UNKNOWN, "(unknown)" },
};
case LLM_ARCH_MINIMAX_M3:
case LLM_ARCH_MISTRAL4:
case LLM_ARCH_KIMI_LINEAR:
+ case LLM_ARCH_QWEN3TTS:
return false;
default:
return true;
LLM_ARCH_MINIMAX_M3,
LLM_ARCH_DFLASH,
LLM_ARCH_NANBEIGE,
+ LLM_ARCH_QWEN3TTS,
LLM_ARCH_UNKNOWN,
};
LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model);
// returns the number of extracted layers from target model
LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model);
+
+// retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab)
+// returns total number of elements or 0 on error
+// if out is nullptr, returns the number of tokens without writing to out
+// caller must allocate enough memory for out before calling
+LLAMA_API uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out);
} else {
throw std::runtime_error(std::string("expecting ',' at ") + pos);
}
- bool has_max = max_times != UINT64_MAX;
- if (min_times > MAX_REPETITION_THRESHOLD || (has_max && max_times > MAX_REPETITION_THRESHOLD)) {
+ if (min_times > MAX_REPETITION_THRESHOLD) {
throw std::runtime_error(std::string("number of repetitions exceeds sane defaults, please reduce the number of repetitions"));
}
+ if (max_times != UINT64_MAX && max_times > MAX_REPETITION_THRESHOLD) {
+ max_times = UINT64_MAX;
+ }
handle_repetitions(min_times, max_times);
} else {
break;
for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {
t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;
GGML_ASSERT(t_meta.ne[dim] >= 1);
- t_meta.nb[dim] = dim == 0 ? ggml_type_size(type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];
+ if (dim == 0) {
+ t_meta.nb[dim] = ggml_type_size(type);
+ } else if (dim == 1) {
+ t_meta.nb[dim] = ggml_row_size(type, t_meta.ne[dim-1]);
+ } else {
+ t_meta.nb[dim] = t_meta.nb[dim-1]*t_meta.ne[dim-1];
+ }
GGML_ASSERT(t_meta.nb[dim] >= 1);
}
ggml_set_name(&t_meta, tn.str().c_str());
if (flags & TENSOR_ALLOW_RESHAPE) {
for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {
t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;
- t_meta.nb[dim] = dim == 0 ? ggml_type_size(t_meta.type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];
+ if (dim == 0) {
+ t_meta.nb[dim] = ggml_type_size(t_meta.type);
+ } else if (dim == 1) {
+ t_meta.nb[dim] = ggml_row_size(t_meta.type, t_meta.ne[dim-1]);
+ } else {
+ t_meta.nb[dim] = t_meta.ne[dim-1]*t_meta.nb[dim-1];
+ }
}
}
+ GGML_ASSERT(ggml_nbytes(&t_meta) == ggml_nbytes(cur));
+
ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta);
if (buft == nullptr) {
return nullptr;
return new llama_model_qwen3vl(params);
case LLM_ARCH_QWEN3VLMOE:
return new llama_model_qwen3vlmoe(params);
+ case LLM_ARCH_QWEN3TTS:
+ return new llama_model_qwen3tts(params);
case LLM_ARCH_PHI2:
return new llama_model_phi2(params);
case LLM_ARCH_PHI3:
case LLM_ARCH_QWEN3VLMOE:
case LLM_ARCH_QWEN35:
case LLM_ARCH_QWEN35MOE:
+ case LLM_ARCH_QWEN3TTS:
return LLAMA_ROPE_TYPE_IMROPE;
case LLM_ARCH_GLM4:
int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,
int flags) {
const int64_t n_embd_qkv = n_embd_q_ + n_embd_k_ + n_embd_v_;
+
+ if (flags & TENSOR_SKIP) {
+ const int skip = TENSOR_NOT_REQUIRED | TENSOR_SKIP;
+
+ create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, skip | TENSOR_SKIP_IF_VIRTUAL);
+ create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, skip | TENSOR_SKIP_IF_VIRTUAL);
+ create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", bid), {n_embd_, n_embd_q_}, skip);
+ create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", bid), {n_embd_, n_embd_k_}, skip);
+ create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", bid), {n_embd_, n_embd_v_}, skip);
+ create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", bid), {n_embd_q_}, skip);
+ create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", bid), {n_embd_k_}, skip);
+ create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", bid), {n_embd_v_}, skip);
+ return;
+ }
+
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
if (layer.wqkv) {
layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
uint32_t llama_model_target_layer_ids_n(const struct llama_model * model) {
return (uint32_t) model->target_layer_ids.size();
}
+
+uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out) {
+ if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr) {
+ return 0;
+ }
+
+ const ggml_tensor * tensor = model->tok_embd;
+ const size_t nelements = ggml_nelements(tensor);
+ GGML_ASSERT(nelements <= UINT32_MAX); // for the return type
+
+ if (out == nullptr) {
+ return (uint32_t) nelements;
+ }
+
+ if (tensor->type == GGML_TYPE_F32) {
+ ggml_backend_tensor_get(tensor, out, 0, nelements * sizeof(float));
+ return (uint32_t) nelements;
+ }
+
+ std::vector<uint8_t> buf(ggml_nbytes(tensor));
+ ggml_backend_tensor_get(tensor, buf.data(), 0, buf.size());
+
+ const ggml_type_traits * traits = ggml_get_type_traits(tensor->type);
+ if (tensor->type == GGML_TYPE_F16) {
+ ggml_fp16_to_fp32_row((const ggml_fp16_t *) buf.data(), out, nelements);
+ } else if (tensor->type == GGML_TYPE_BF16) {
+ ggml_bf16_to_fp32_row((const ggml_bf16_t *) buf.data(), out, nelements);
+ } else if (ggml_is_quantized(tensor->type) && traits->to_float != nullptr) {
+ traits->to_float(buf.data(), out, nelements);
+ } else {
+ GGML_ABORT("unsupported tensor type for dequantization: %s", ggml_type_name(tensor->type));
+ }
+
+ return (uint32_t) nelements;
+}
// DRY
struct llama_sampler_dry {
- int32_t total_context_size;
-
const float dry_multiplier;
const float dry_base;
const int32_t dry_allowed_length;
return;
}
- int32_t effective_dry_penalty_last_n = (ctx->dry_penalty_last_n == -1) ? ctx->total_context_size : std::max(ctx->dry_penalty_last_n, 0);
- int last_n_repeat = std::min(std::min((int)ctx->last_tokens.size(), effective_dry_penalty_last_n), ctx->total_context_size);
+ int last_n_repeat = std::min((int) ctx->last_tokens.size(), ctx->dry_penalty_last_n);
if (last_n_repeat <= ctx->dry_allowed_length) {
return;
llama_vocab dummy_vocab;
// dummy vocab is passed because it is only needed for raw sequence breaker processing, which we have already done and will simply be copying
- auto * result = llama_sampler_init_dry(&dummy_vocab, ctx->total_context_size, ctx->dry_multiplier, ctx->dry_base, ctx->dry_allowed_length, ctx->dry_penalty_last_n, NULL, 0);
+ auto * result = llama_sampler_init_dry(&dummy_vocab, ctx->dry_multiplier, ctx->dry_base, ctx->dry_allowed_length, ctx->dry_penalty_last_n, NULL, 0);
// Copy the state, including the processed breakers
{
/* .backend_set_input = */ nullptr,
};
-struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab, int32_t n_ctx_train, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const char** seq_breakers, size_t num_breakers) {
- int32_t effective_dry_penalty_last_n = (dry_penalty_last_n == -1) ? n_ctx_train : std::max(dry_penalty_last_n, 0);
+struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const char** seq_breakers, size_t num_breakers) {
+ dry_penalty_last_n = std::max(dry_penalty_last_n, 0);
std::unordered_multimap<llama_token, std::vector<llama_token>> processed_breakers;
const int MAX_CHAR_LEN = 40;
const int MAX_SEQ_LEN = 20;
return llama_sampler_init(
/* .iface = */ &llama_sampler_dry_i,
/* .ctx = */ new llama_sampler_dry {
- /* .total_context_size = */ n_ctx_train,
/* .dry_multiplier = */ dry_multiplier,
/* .dry_base = */ dry_base,
/* .dry_allowed_length = */ dry_allowed_length,
/* .dry_penalty_last_n = */ dry_penalty_last_n,
/* .dry_processed_breakers = */ std::move(processed_breakers),
- /* .dry_repeat_count = */ dry_enabled ? std::vector<int>(effective_dry_penalty_last_n, 0) : std::vector<int>{},
+ /* .dry_repeat_count = */ dry_enabled ? std::vector<int>(dry_penalty_last_n, 0) : std::vector<int>{},
/* .dry_max_token_repeat = */ {},
- /* .last_tokens = */ dry_enabled ? ring_buffer<llama_token>(effective_dry_penalty_last_n) : ring_buffer<llama_token>(0),
+ /* .last_tokens = */ dry_enabled ? ring_buffer<llama_token>(dry_penalty_last_n) : ring_buffer<llama_token>(0),
}
);
}
// wrapper for test-sampling.cpp
-struct llama_sampler * llama_sampler_init_dry_testing(int32_t context_size, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const std::vector<std::vector<llama_token>>& seq_breakers) {
+struct llama_sampler * llama_sampler_init_dry_testing(float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const std::vector<std::vector<llama_token>>& seq_breakers) {
llama_vocab dummy_vocab;
- auto * result = llama_sampler_init_dry(&dummy_vocab, context_size, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, NULL, 0);
+ auto * result = llama_sampler_init_dry(&dummy_vocab, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, NULL, 0);
auto * ctx = (llama_sampler_dry *) result->ctx;
// Process the token-based sequence breakers
};
struct llama_sampler * llama_sampler_init_dry_testing(
- int32_t context_size,
float dry_multiplier,
float dry_base,
int32_t dry_allowed_length,
if (vocab.is_byte(token_id)) {
if (entry.text.length() == 6 && entry.text.substr(0, 3) == "<0x" && entry.text.back() == '>') {
std::string hex_str = entry.text.substr(3, 2);
- int byte_val = std::stoi(hex_str, nullptr, 16);
- bytes_[byte_val] = static_cast<llama_token>(token_id);
+ if (std::isxdigit(static_cast<unsigned char>(hex_str[0])) && std::isxdigit(static_cast<unsigned char>(hex_str[1]))) {
+ int byte_val = std::stoi(hex_str, nullptr, 16);
+ bytes_[byte_val] = static_cast<llama_token>(token_id);
+ }
}
continue;
}
if (vocab.is_byte(token)) {
// Handle byte tokens like <0xXX>
if (token_text.length() == 6 && token_text.substr(0, 3) == "<0x" && token_text.back() == '>') {
- int hex_val = std::stoi(token_text.substr(3, 2), nullptr, 16);
- if (length < 1) {
- return -1;
+ std::string hex_str = token_text.substr(3, 2);
+ if (std::isxdigit(static_cast<unsigned char>(hex_str[0])) && std::isxdigit(static_cast<unsigned char>(hex_str[1]))) {
+ int hex_val = std::stoi(hex_str, nullptr, 16);
+ if (length < 1) {
+ return -1;
+ }
+ buf[0] = static_cast<char>(hex_val);
+ return 1;
}
- buf[0] = static_cast<char>(hex_val);
- return 1;
}
}
/// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first.
LLAMA_API struct llama_sampler * llama_sampler_init_penalties(
int32_t n_vocab,
- int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size)
+ int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty)
float penalty_repeat, // must be > 0.0, 1.0 = disabled
float penalty_freq, // must be finite, 0.0 = disabled
float penalty_present); // must be finite, 0.0 = disabled
/// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982
LLAMA_API struct llama_sampler * llama_sampler_init_dry(
const struct llama_vocab * vocab,
- int32_t n_ctx_train,
float dry_multiplier,
float dry_base,
int32_t dry_allowed_length,
- int32_t dry_penalty_last_n,
+ int32_t dry_penalty_last_n, // last n tokens to penalize (0 = disable penalty)
const char ** seq_breakers,
size_t num_breakers);
layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, 0);
layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, 0);
layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, 0);
- layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank * o_groups}, 0);
+ layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank, o_groups}, TENSOR_ALLOW_RESHAPE);
layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, 0);
layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0);
};
+struct llama_model_qwen3tts : public llama_model_qwen3vl {
+ llama_model_qwen3tts(const struct llama_model_params & params) : llama_model_qwen3vl(params) {}
+};
+
+
struct llama_model_phi2 : public llama_model_base {
llama_model_phi2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
--- /dev/null
+#include "models.h"
+
+// llama_model_qwen3tts reuses llama_model_qwen3vl's hparams/tensors/graph logic
void llama_model_qwen3vl::load_arch_tensors(llama_model_loader &) {
LLAMA_LOAD_LOCALS;
+ int64_t n_vocab_out = n_vocab;
+ if (arch == LLM_ARCH_QWEN3TTS) {
+ n_vocab_out = 3072;
+ }
+
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
// output
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
- output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab_out}, TENSOR_NOT_REQUIRED);
// if output is NULL, init from the input tok embed
if (output == NULL) {
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
// lm_head
cur = build_lora_mm(model.output, cur, model.output_s);
+ int64_t n_vocab_in = model.tok_embd->ne[1];
+ int64_t n_vocab_out = model.output->ne[1];
+ if (n_vocab_in > n_vocab_out) {
+ // case: Qwen3TTS model with codec_head as output
+ GGML_ASSERT(model.output_norm);
+ int64_t pad = n_vocab_in - n_vocab_out;
+
+ // using this trick to get a scalar -inf tensor to pad the output
+ ggml_tensor * neg_inf = ggml_scale_bias(ctx0,
+ ggml_view_1d(ctx0, model.output_norm, 1, 0),
+ 0.0f, -INFINITY);
+ neg_inf = ggml_repeat_4d(ctx0, neg_inf, pad, cur->ne[1], 1, 1);
+ cur = ggml_concat(ctx0, neg_inf, cur, 0); // [padded .. n_vocab_out, n_stream]
+
+ } else if (n_vocab_in < n_vocab_out) {
+ GGML_ABORT("invalid case");
+ }
+
cb(cur, "result_output", -1);
res->t_logits = cur;
overview / pkg / ggml / sources / whisper.cpp / commitdiff