self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"])
self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"])
self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"])
- self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"])
+ if self.model_arch == gguf.MODEL_ARCH.DEEPSEEK4:
+ self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"])
if self.mtp_only and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers", 0)) > 0:
self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
super().prepare_tensors()
self._is_mxfp4 = True
self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE
+
+
+@ModelBase.register("DeepseekV4DSparkModel")
+class DeepseekV4DSparkModel(DeepseekV4Model):
+ model_arch = gguf.MODEL_ARCH.DFLASH
+
+ _DSPARK_ROOT_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {
+ "main_proj.weight": (gguf.MODEL_TENSOR.FC, ".weight"),
+ "main_norm.weight": (gguf.MODEL_TENSOR.ENC_OUTPUT_NORM, ".weight"),
+ "markov_head.markov_w1.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W1, ".weight"),
+ "markov_head.markov_w2.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W2, ".weight"),
+ "confidence_head.proj.weight": (gguf.MODEL_TENSOR.DSPARK_CONF_PROJ, ".weight"),
+ }
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ self.block_count = 1 + max(
+ int(match.group(1)) for name in self.model_tensors
+ if (match := re.match(r"layers\.(\d+)\.", name))
+ )
+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+ self.hparams["compress_ratios"] = [0] * self.block_count
+ self.hparams["num_hash_layers"] = 0
+
+ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+ if remote_hf_model_id is None:
+ return super().index_tensors()
+
+ with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f:
+ weight_map = json.load(f)["weight_map"]
+
+ part_names = sorted({
+ part_name for name, part_name in weight_map.items()
+ if name.startswith("mtp.")
+ })
+ tensors: dict[str, Callable[[], Tensor]] = {}
+
+ for part_name in part_names:
+ from huggingface_hub import hf_hub_download
+
+ logger.info("gguf: caching remote DSpark part '%s'", part_name)
+ part_path = Path(hf_hub_download(repo_id=remote_hf_model_id, filename=part_name))
+ with gguf.utility.SafetensorsLocal(part_path) as model_part:
+ for name in model_part:
+ data = model_part[name]
+ data_gen = lambda data=data: LazyTorchTensor.from_local_tensor(data) # noqa: E731
+ if titem := self.filter_tensors((name, data_gen)):
+ tensor_name, tensor_gen = titem
+ tensors[tensor_name] = tensor_gen
+
+ return tensors
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, gen = item
+ if not name.startswith("mtp."):
+ return None
+ return super().filter_tensors((cls._rekey_mtp_tensor_name(name), gen))
+
+ @staticmethod
+ def _rekey_mtp_tensor_name(name: str) -> str:
+ match = re.match(r"mtp\.(\d+)\.(.+)$", name)
+ if match is None:
+ raise ValueError(f"Unexpected DSpark tensor {name!r}")
+
+ stage, rest = match.group(1), match.group(2)
+ root_names = (
+ "main_proj.scale",
+ "norm.weight",
+ "hc_head_fn",
+ "hc_head_base",
+ "hc_head_scale",
+ )
+ if rest in DeepseekV4DSparkModel._DSPARK_ROOT_MAP or rest in root_names:
+ return rest
+ return f"layers.{stage}.{rest}"
+
+ def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:
+ if name in self._DSPARK_ROOT_MAP:
+ return self._DSPARK_ROOT_MAP[name]
+ return super()._map_dsv4_tensor_name(name, bid)
+
+ def set_vocab(self):
+ if self.target_model_dir is None:
+ raise ValueError("DeepSeek-V4 DSpark requires --target-model-dir with the target tokenizer")
+
+ original_dir = self.dir_model
+ try:
+ self.dir_model = self.target_model_dir
+ super().set_vocab()
+ finally:
+ self.dir_model = original_dir
+
+ self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"])
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ self.gguf_writer.add_block_size(self.hparams["dspark_block_size"])
+ self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]])
help="Export only the multi-token prediction (MTP) head as a separate GGUF, suitable for use as a speculative draft. An 'mtp-' prefix will be added to the output file name.",
)
parser.add_argument(
- "--no-mtp", action="store_true",
- help="Exclude the multi-token prediction (MTP) head from the converted GGUF. Pair with --mtp on a second run to publish trunk and MTP as two files. Note: the split form duplicates embeddings, but even though the bundled default is more space-efficient overall, this allows differing quantization which may be more performant.",
+ "--no-nextn", "--no-mtp", dest="no_mtp", action="store_true",
+ help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.",
+ )
+ parser.add_argument(
+ "--dspark", action="store_true",
+ help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.",
)
parser.add_argument(
"--mistral-format", action="store_true",
from conversion.mistral import MistralModel
model_class = MistralModel
- if args.mtp and args.no_mtp:
- logger.error("--mtp and --no-mtp are mutually exclusive")
+ if sum((args.mtp, args.no_mtp, args.dspark)) > 1:
+ logger.error("--mtp, --no-nextn, and --dspark are mutually exclusive")
sys.exit(1)
+ if args.dspark:
+ if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM":
+ logger.error("--dspark is only supported for DeepseekV4ForCausalLM")
+ sys.exit(1)
+ from conversion.deepseek import DeepseekV4DSparkModel
+ model_class = DeepseekV4DSparkModel
+
if args.mtp or args.no_mtp:
if not model_class.supports_mtp_export:
- logger.error("--mtp / --no-mtp are not supported for %s", model_architecture)
+ logger.error("--mtp / --no-nextn are not supported for %s", model_architecture)
sys.exit(1)
if args.no_mtp:
model_class.no_mtp = True
MODEL_TENSOR.ATTN_OUT,
MODEL_TENSOR.ATTN_Q_NORM,
MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_SINKS,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV,
+ MODEL_TENSOR.ATTN_KV_NORM,
+ MODEL_TENSOR.ATTN_OUT_A,
+ MODEL_TENSOR.ATTN_OUT_B,
+ MODEL_TENSOR.HC_ATTN_FN,
+ MODEL_TENSOR.HC_ATTN_BASE,
+ MODEL_TENSOR.HC_ATTN_SCALE,
+ MODEL_TENSOR.HC_FFN_FN,
+ MODEL_TENSOR.HC_FFN_BASE,
+ MODEL_TENSOR.HC_FFN_SCALE,
+ MODEL_TENSOR.HC_HEAD_FN,
+ MODEL_TENSOR.HC_HEAD_BASE,
+ MODEL_TENSOR.HC_HEAD_SCALE,
MODEL_TENSOR.FFN_NORM,
MODEL_TENSOR.FFN_GATE,
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
MODEL_TENSOR.FC,
MODEL_TENSOR.ENC_OUTPUT_NORM,
# optional DSpark heads
overview / pkg / ggml / sources / llama.cpp / commitdiff