From: Kerfuffle Date: Wed, 6 Sep 2023 08:49:11 +0000 (-0600) Subject: convert-llama-ggml-to-gguf: Try to handle files older than GGJTv3 (#3023) X-Git-Tag: gguf-v0.4.0~120 X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=ea2c85d5d2a93d39d0172222917f3195f0e456ff;p=pkg%2Fggml%2Fsources%2Fllama.cpp convert-llama-ggml-to-gguf: Try to handle files older than GGJTv3 (#3023) * convert-llama-ggmlv3-to-gguf: Try to handle files older than GGJTv3 * Better error messages for files that cannot be converted * Add file type to GGUF output * Rename to convert-llama-ggml-to-gguf.py * Include original file type information in description * Improve some informational output --- diff --git a/convert-llama-ggml-to-gguf.py b/convert-llama-ggml-to-gguf.py new file mode 100755 index 00000000..b5d3e0b3 --- /dev/null +++ b/convert-llama-ggml-to-gguf.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import math +import struct +import sys +from enum import IntEnum +from pathlib import Path + +import numpy as np + +import os +if 'NO_LOCAL_GGUF' not in os.environ: + sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf')) +import gguf + +# Note: Does not support GGML_QKK_64 +QK_K = 256 +# Items here are (block size, type size) +GGML_QUANT_SIZES = { + gguf.GGMLQuantizationType.F32 : (1, 4), + gguf.GGMLQuantizationType.F16 : (1, 2), + gguf.GGMLQuantizationType.Q4_0 : (32, 2 + 16), + gguf.GGMLQuantizationType.Q4_1 : (32, 2 + 2 + 16), + gguf.GGMLQuantizationType.Q5_0 : (32, 2 + 4 + 16), + gguf.GGMLQuantizationType.Q5_1 : (32, 2 + 2 + 4 + 16), + gguf.GGMLQuantizationType.Q8_0 : (32, 2 + 32), + gguf.GGMLQuantizationType.Q8_1 : (32, 4 + 4 + 32), + gguf.GGMLQuantizationType.Q2_K : (256, 2 + 2 + QK_K // 16 + QK_K // 4), + gguf.GGMLQuantizationType.Q3_K : (256, 2 + QK_K // 4 + QK_K // 8 + 12), + gguf.GGMLQuantizationType.Q4_K : (256, 2 + 2 + QK_K // 2 + 12), + gguf.GGMLQuantizationType.Q5_K : (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12), + gguf.GGMLQuantizationType.Q6_K : (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16), + gguf.GGMLQuantizationType.Q8_K : (256, 4 + QK_K + QK_K // 8), +} + +class GGMLFormat(IntEnum): + GGML = 0 + GGMF = 1 + GGJT = 2 + +class GGMLFType(IntEnum): + ALL_F32 = 0 + MOSTLY_F16 = 1 + MOSTLY_Q4_0 = 2 + MOSTLY_Q4_1 = 3 + MOSTLY_Q4_1_SOME_F16 = 4 + MOSTLY_Q8_0 = 7 + MOSTLY_Q5_0 = 8 + MOSTLY_Q5_1 = 9 + MOSTLY_Q2_K = 10 + MOSTLY_Q3_K_S = 11 + MOSTLY_Q3_K_M = 12 + MOSTLY_Q3_K_L = 13 + MOSTLY_Q4_K_S = 14 + MOSTLY_Q4_K_M = 15 + MOSTLY_Q5_K_S = 16 + MOSTLY_Q5_K_M = 17 + MOSTLY_Q6_K = 18 + +class Hyperparameters: + def __init__(self): + self.n_vocab = self.n_embd = self.n_mult = self.n_head = 0 + self.n_layer = self.n_rot = self.n_ff = 0 + self.ftype = GGMLFType.ALL_F32 + + def set_n_ff(self, model): + ff_tensor_idx = model.tensor_map.get(b'layers.0.feed_forward.w1.weight') + assert ff_tensor_idx is not None, 'Missing layer 0 FF tensor' + ff_tensor = model.tensors[ff_tensor_idx] + self.n_ff = ff_tensor.dims[1] + + def load(self, data, offset): + ( + self.n_vocab, + self.n_embd, + self.n_mult, + self.n_head, + self.n_layer, + self.n_rot, + ftype, + ) = struct.unpack('<7I', data[offset:offset + (4 * 7)]) + try: + self.ftype = GGMLFType(ftype) + except ValueError: + raise ValueError(f'Invalid ftype {ftype}') + return 4 * 7 + + def __str__(self): + return f'' + +class Vocab: + def __init__(self, load_scores = True): + self.items = [] + self.load_scores = load_scores + + def load(self, data, offset, n_vocab): + orig_offset = offset + for _ in range(n_vocab): + itemlen = struct.unpack('= 0 and n_dims <= 4, f'Invalid tensor dimensions {n_dims}' + assert name_len < 4096, 'Absurd tensor name length' + quant = GGML_QUANT_SIZES.get(dtype) + assert quant is not None, 'Unknown tensor type' + (blksize, tysize) = quant + offset += 12 + self.dtype= dtype + self.dims = struct.unpack(f'<{n_dims}I', data[offset:offset + (4 * n_dims)]) + offset += 4 * n_dims + self.name = bytes(data[offset:offset + name_len]) + offset += name_len + pad = ((offset + 31) & ~31) - offset if self.use_padding else 0 + offset += pad + n_elems = np.prod(self.dims) + n_bytes = np.int64(np.int64(n_elems) * np.int64(tysize)) // np.int64(blksize) + self.start_offset = offset + self.len_bytes = n_bytes + offset += n_bytes + # print(n_dims, name_len, dtype, self.dims, self.name, pad) + return offset - orig_offset + +class GGMLModel: + def __init__(self): + self.hyperparameters = None + self.vocab = None + self.tensor_map = {} + self.tensors = [] + + def validate_header(self, data, offset): + magic = bytes(data[offset:offset + 4]) + if magic == b'GGUF': + raise ValueError('File is already in GGUF format.') + if magic == b'lmgg': + self.file_format = GGMLFormat.GGML + self.format_version = 1 + return 4 + version = struct.unpack(' 3: + raise ValueError(f'Cannot handle unexpected GGJT file version {version}') + self.file_format = GGMLFormat.GGJT + self.format_version = version + return 8 + raise ValueError(f"Unexpected file magic {magic!r}! This doesn't look like a GGML format file.") + + def validate_conversion(self, ftype): + err = '' + if (self.file_format < GGMLFormat.GGJT or self.format_version < 2): + if ftype not in (GGMLFType.ALL_F32, GGMLFType.MOSTLY_F16): + err = 'Quantizations changed in GGJTv2. Can only convert unquantized GGML files older than GGJTv2.' + elif (self.file_format == GGMLFormat.GGJT and self.format_version == 2): + if ftype in ( GGMLFType.MOSTLY_Q4_0, GGMLFType.MOSTLY_Q4_1, + GGMLFType.MOSTLY_Q4_1_SOME_F16, GGMLFType.MOSTLY_Q8_0): + err = 'Q4 and Q8 quantizations changed in GGJTv3.' + if len(err) > 0: + raise ValueError(f'{err} Sorry, your {self.file_format.name}v{self.format_version} file of type {ftype.name} is not eligible for conversion.') + + def load(self, data, offset): + offset += self.validate_header(data, offset) + hp = Hyperparameters() + offset += hp.load(data, offset) + print(f'* File format: {self.file_format.name}v{self.format_version} with ftype {hp.ftype.name}') + self.validate_conversion(hp.ftype) + vocab = Vocab(load_scores = self.file_format > GGMLFormat.GGML) + offset += vocab.load(data, offset, hp.n_vocab) + tensors: list[Tensor] = [] + tensor_map = {} + while offset < len(data): + tensor = Tensor(use_padding = self.file_format > GGMLFormat.GGMF) + offset += tensor.load(data, offset) + tensor_map[tensor.name] = len(tensors) + tensors.append(tensor) + self.hyperparameters = hp + self.vocab = vocab + self.tensors = tensors + self.tensor_map = tensor_map + hp.set_n_ff(self) + return offset + +class GGMLToGGUF: + def __init__(self, ggml_model, data, cfg, params_override = None, vocab_override = None, special_vocab = None): + hp = ggml_model.hyperparameters + self.model = ggml_model + self.data = data + self.cfg = cfg + self.params_override = params_override + self.vocab_override = vocab_override + self.special_vocab = special_vocab + if params_override is not None: + n_kv_head = params_override.n_head_kv + else: + if cfg.gqa == 1: + n_kv_head = hp.n_head + else: + gqa = float(cfg.gqa) + n_kv_head = None + for x in range(1, 256): + if float(hp.n_head) / float(x) == gqa: + n_kv_head = x + assert n_kv_head is not None, "Couldn't determine n_kv_head from GQA param" + print(f'- Guessed n_kv_head = {n_kv_head} based on GQA {cfg.gqa}') + self.n_kv_head = n_kv_head + self.name_map = gguf.get_tensor_name_map(gguf.MODEL_ARCH.LLAMA, ggml_model.hyperparameters.n_layer) + + def save(self): + print('* Preparing to save GGUF file') + gguf_writer = gguf.GGUFWriter( + self.cfg.output, + gguf.MODEL_ARCH_NAMES[gguf.MODEL_ARCH.LLAMA], + use_temp_file = False ) + self.add_params(gguf_writer) + self.add_vocab(gguf_writer) + if self.special_vocab is not None: + self.special_vocab.add_to_gguf(gguf_writer) + self.add_tensors(gguf_writer) + print(" gguf: write header") + gguf_writer.write_header_to_file() + print(" gguf: write metadata") + gguf_writer.write_kv_data_to_file() + print(" gguf: write tensors") + gguf_writer.write_tensors_to_file() + gguf_writer.close() + + def add_params(self, gguf_writer): + hp = self.model.hyperparameters + cfg = self.cfg + if cfg.desc is not None: + desc = cfg.desc + else: + desc = f'converted from legacy {self.model.file_format.name}v{self.model.format_version} {hp.ftype.name} format' + try: + # Filenames aren't necessarily valid UTF8. + name = cfg.name if cfg.name is not None else cfg.input.name + except UnicodeDecodeError: + name = None + print('* Adding model parameters and KV items') + if name is not None: + gguf_writer.add_name(name) + gguf_writer.add_description(desc) + gguf_writer.add_file_type(int(hp.ftype)) + if self.params_override is not None: + po = self.params_override + assert po.n_embd == hp.n_embd, 'Model hyperparams mismatch' + assert po.n_layer == hp.n_layer, 'Model hyperparams mismatch' + assert po.n_head == hp.n_head, 'Model hyperparams mismatch' + gguf_writer.add_context_length (po.n_ctx) + gguf_writer.add_embedding_length (po.n_embd) + gguf_writer.add_block_count (po.n_layer) + gguf_writer.add_feed_forward_length (po.n_ff) + gguf_writer.add_rope_dimension_count(po.n_embd // po.n_head) + gguf_writer.add_head_count (po.n_head) + gguf_writer.add_head_count_kv (po.n_head_kv) + gguf_writer.add_layer_norm_rms_eps (po.f_norm_eps) + return + gguf_writer.add_context_length(cfg.context_length) + gguf_writer.add_embedding_length(hp.n_embd) + gguf_writer.add_block_count(hp.n_layer) + gguf_writer.add_feed_forward_length(hp.n_ff) + gguf_writer.add_rope_dimension_count(hp.n_embd // hp.n_head) + gguf_writer.add_head_count(hp.n_head) + gguf_writer.add_head_count_kv(self.n_kv_head) + gguf_writer.add_layer_norm_rms_eps(float(cfg.eps)) + + def add_vocab(self, gguf_writer): + hp = self.model.hyperparameters + gguf_writer.add_tokenizer_model('llama') + tokens = [] + scores = [] + toktypes = [] + if self.vocab_override is not None: + vo = self.vocab_override + print('* Adding vocab item(s)') + for (idx, (vbytes, score, ttype)) in enumerate(vo.all_tokens()): + tokens.append(vbytes) + scores.append(score) + toktypes.append(ttype) + assert len(tokens) == hp.n_vocab, \ + f'Override vocab has a different number of items than hyperparameters - override = {len(tokens)} but n_vocab={hp.n_vocab}' + gguf_writer.add_token_list(tokens) + gguf_writer.add_token_scores(scores) + if len(toktypes) > 0: + gguf_writer.add_token_types(toktypes) + return + print(f'* Adding {hp.n_vocab} vocab item(s)') + assert len(self.model.vocab.items) >= 3, 'Cannot handle unexpectedly short model vocab' + for (tokid, (vbytes, vscore)) in enumerate(self.model.vocab.items): + tt = 1 # Normal + # Special handling for UNK, BOS, EOS tokens. + if tokid <= 2: + if tokid == 0: + vbytes = b'' + tt = 2 + elif tokid == 1: + vbytes = b'' + tt = 3 + else: + vbytes = b'' + tt = 3 + elif len(vbytes) == 0: + tt = 3 # Control + elif tokid >= 3 and tokid <= 258 and len(vbytes) == 1: + vbytes = bytes(f'<0x{vbytes[0]:02X}>', encoding = 'UTF-8') + tt = 6 # Byte + else: + vbytes = vbytes.replace(b' ', b'\xe2\x96\x81') + toktypes.append(tt) + tokens.append(vbytes) + scores.append(vscore) + gguf_writer.add_token_list(tokens) + gguf_writer.add_token_scores(scores) + gguf_writer.add_token_types(toktypes) + gguf_writer.add_unk_token_id(0) + gguf_writer.add_bos_token_id(1) + gguf_writer.add_eos_token_id(2) + + def add_tensors(self, gguf_writer): + tensor_map = self.name_map + data = self.data + print(f'* Adding {len(self.model.tensors)} tensor(s)') + for tensor in self.model.tensors: + name = str(tensor.name, 'UTF-8') + mapped_name = tensor_map.get_name(name, try_suffixes = (".weight", ".bias")) + assert mapped_name is not None, f'Bad name {name}' + tempdims = list(tensor.dims[:]) + if len(tempdims) > 1: + temp = tempdims[1] + tempdims[1] = tempdims[0] + tempdims[0] = temp + # print(f'+ {tensor.name} | {mapped_name} {tensor.dims} :: {tempdims}') + gguf_writer.add_tensor( + mapped_name, + data[tensor.start_offset:tensor.start_offset + tensor.len_bytes], + raw_shape = tempdims, + raw_dtype = tensor.dtype ) + +def handle_metadata(cfg, hp): + import convert + assert cfg.model_metadata_dir.is_dir(), 'Metadata dir is not a directory' + hf_config_path = cfg.model_metadata_dir / "config.json" + orig_config_path = cfg.model_metadata_dir / "params.json" + # We pass a fake model here. "original" mode will check the shapes of some + # tensors if information is missing in the .json file: other than that, the + # model data isn't used so this should be safe (at least for now). + fakemodel = { + 'tok_embeddings.weight': convert.LazyTensor.__new__(convert.LazyTensor), + 'layers.0.feed_forward.w1.weight': convert.LazyTensor.__new__(convert.LazyTensor), + } + fakemodel['tok_embeddings.weight'].shape = [hp.n_vocab] + fakemodel['layers.0.feed_forward.w1.weight'].shape = [hp.n_ff] + if hf_config_path.exists(): + params = convert.Params.loadHFTransformerJson(fakemodel, hf_config_path) + elif orig_config_path.exists(): + params = convert.Params.loadOriginalParamsJson(fakemodel, orig_config_path) + else: + raise ValueError('Unable to load metadata') + vocab = convert.load_vocab( + cfg.vocab_dir if cfg.vocab_dir is not None else cfg.model_metadata_dir, + cfg.vocabtype ) + # FIXME: Respect cfg.vocab_dir? + svocab = gguf.SpecialVocab(cfg.model_metadata_dir) + convert.check_vocab_size(params, vocab) + return (params, vocab, svocab) + +def handle_args(): + parser = argparse.ArgumentParser(description = 'Convert GGML models to GGUF') + parser.add_argument('--input', '-i', type = Path, required = True, + help = 'Input GGMLv3 filename') + parser.add_argument('--output', '-o', type = Path, required = True, + help ='Output GGUF filename') + parser.add_argument('--name', + help = 'Set model name') + parser.add_argument('--desc', + help = 'Set model description') + parser.add_argument('--gqa', type = int, default = 1, + help = 'grouped-query attention factor (use 8 for LLaMA2 70B)') + parser.add_argument('--eps', default = '5.0e-06', + help = 'RMS norm eps: Use 1e-6 for LLaMA1 and OpenLLaMA, use 1e-5 for LLaMA2') + parser.add_argument('--context-length', '-c', type=int, default = 2048, + help = 'Default max context length: LLaMA1 is typically 2048, LLaMA2 is typically 4096') + parser.add_argument('--model-metadata-dir', '-m', type = Path, + help ='Load HuggingFace/.pth vocab and metadata from the specified directory') + parser.add_argument("--vocab-dir", type=Path, + help="directory containing tokenizer.model, if separate from model file - only meaningful with --model-metadata-dir") + parser.add_argument("--vocabtype", choices=["spm", "bpe"], default="spm", + help="vocab format - only meaningful with --model-metadata-dir and/or --vocab-dir (default: spm)") + return parser.parse_args() + +def main(): + cfg = handle_args() + print(f'* Using config: {cfg}') + print('\n=== WARNING === Be aware that this conversion script is best-effort. Use a native GGUF model if possible. === WARNING ===\n') + if cfg.model_metadata_dir is None and (cfg.gqa == 1 or cfg.eps == '5.0e-06'): + print('- Note: If converting LLaMA2, specifying "--eps 1e-5" is required. 70B models also need "--gqa 8".') + data = np.memmap(cfg.input, mode = 'r') + model = GGMLModel() + print('* Scanning GGML input file') + offset = model.load(data, 0) + print(f'* GGML model hyperparameters: {model.hyperparameters}') + vocab_override = None + params_override = None + special_vocab = None + if cfg.model_metadata_dir is not None: + (params_override, vocab_override, special_vocab) = handle_metadata(cfg, model.hyperparameters) + print('!! Note: When overriding params the --gqa, --eps and --context-length options are ignored.') + print(f'* Overriding params: {params_override}') + print(f'* Overriding vocab: {vocab_override}') + print(f'* Special vocab: {special_vocab}') + else: + print('\n=== WARNING === Special tokens may not be converted correctly. Use --model-metadata-dir if possible === WARNING ===\n') + if model.file_format == GGMLFormat.GGML: + print('! This is a very old GGML file that does not contain vocab scores. Strongly recommend using model metadata!') + converter = GGMLToGGUF(model, data, cfg, + params_override = params_override, + vocab_override = vocab_override, + special_vocab = special_vocab ) + converter.save() + print(f'* Successful completion. Output saved to: {cfg.output}') + +if __name__ == '__main__': + main() diff --git a/convert-llama-ggmlv3-to-gguf.py b/convert-llama-ggmlv3-to-gguf.py deleted file mode 100755 index 08ba0c49..00000000 --- a/convert-llama-ggmlv3-to-gguf.py +++ /dev/null @@ -1,353 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import math -import struct -import sys -from pathlib import Path - -import numpy as np - -import os -if 'NO_LOCAL_GGUF' not in os.environ: - sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf')) -import gguf - -# Note: Does not support GGML_QKK_64 -QK_K = 256 -# Items here are (block size, type size) -GGML_QUANT_SIZES = { - gguf.GGMLQuantizationType.F32 : (1, 4), - gguf.GGMLQuantizationType.F16 : (1, 2), - gguf.GGMLQuantizationType.Q4_0 : (32, 2 + 16), - gguf.GGMLQuantizationType.Q4_1 : (32, 2 + 2 + 16), - gguf.GGMLQuantizationType.Q5_0 : (32, 2 + 4 + 16), - gguf.GGMLQuantizationType.Q5_1 : (32, 2 + 2 + 4 + 16), - gguf.GGMLQuantizationType.Q8_0 : (32, 2 + 32), - gguf.GGMLQuantizationType.Q8_1 : (32, 4 + 4 + 32), - gguf.GGMLQuantizationType.Q2_K : (256, 2 + 2 + QK_K // 16 + QK_K // 4), - gguf.GGMLQuantizationType.Q3_K : (256, 2 + QK_K // 4 + QK_K // 8 + 12), - gguf.GGMLQuantizationType.Q4_K : (256, 2 + 2 + QK_K // 2 + 12), - gguf.GGMLQuantizationType.Q5_K : (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12), - gguf.GGMLQuantizationType.Q6_K : (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16), - gguf.GGMLQuantizationType.Q8_K : (256, 4 + QK_K + QK_K // 8), -} - -class Hyperparameters: - def __init__(self): - self.n_vocab = self.n_embd = self.n_mult = self.n_head = self.n_layer = self.n_rot = self.ftype = 0 - self.n_ff = 0 - - def set_n_ff(self, model): - ff_tensor_idx = model.tensor_map.get(b'layers.0.feed_forward.w1.weight') - assert ff_tensor_idx is not None, 'Missing layer 0 FF tensor' - ff_tensor = model.tensors[ff_tensor_idx] - self.n_ff = ff_tensor.dims[1] - - def load(self, data, offset): - ( - self.n_vocab, - self.n_embd, - self.n_mult, - self.n_head, - self.n_layer, - self.n_rot, - self.ftype, - ) = struct.unpack('<7I', data[offset:offset + (4 * 7)]) - return 4 * 7 - - def __str__(self): - return f'' - -class Vocab: - def __init__(self): - self.items = [] - - def load(self, data, offset, n_vocab): - orig_offset = offset - for _ in range(n_vocab): - itemlen = struct.unpack('= 0 and n_dims <= 4, f'Invalid tensor dimensions {n_dims}' - assert name_len < 4096, 'Absurd tensor name length' - quant = GGML_QUANT_SIZES.get(dtype) - assert quant is not None, 'Unknown tensor type' - (blksize, tysize) = quant - offset += 12 - self.dtype= dtype - self.dims = struct.unpack(f'<{n_dims}I', data[offset:offset + (4 * n_dims)]) - offset += 4 * n_dims - self.name = bytes(data[offset:offset + name_len]) - offset += name_len - pad = ((offset + 31) & ~31) - offset - offset += pad - n_elems = np.prod(self.dims) - n_bytes = np.int64(np.int64(n_elems) * np.int64(tysize)) // np.int64(blksize) - self.start_offset = offset - self.len_bytes = n_bytes - offset += n_bytes - # print(n_dims, name_len, dtype, self.dims, self.name, pad) - return offset - orig_offset - -class GGMLV3Model: - def __init__(self): - self.hyperparameters = None - self.vocab = None - self.tensor_map = {} - self.tensors = [] - - def validate_header(self, data, offset): - if bytes(data[offset:offset + 4]) != b'tjgg' or struct.unpack(' 0: - gguf_writer.add_token_types(toktypes) - return - print(f'* Adding {hp.n_vocab} vocab item(s)') - assert len(self.model.vocab.items) >= 3, 'Cannot handle unexpectedly short model vocab' - for (tokid, (vbytes, vscore)) in enumerate(self.model.vocab.items): - tt = 1 # Normal - # Special handling for UNK, BOS, EOS tokens. - if tokid <= 2: - if tokid == 0: - vbytes = b'' - tt = 2 - elif tokid == 1: - vbytes = b'' - tt = 3 - else: - vbytes = b'' - tt = 3 - elif len(vbytes) == 0: - tt = 3 # Control - elif tokid >= 3 and tokid <= 258 and len(vbytes) == 1: - vbytes = bytes(f'<0x{vbytes[0]:02X}>', encoding = 'UTF-8') - tt = 6 # Byte - else: - vbytes = vbytes.replace(b' ', b'\xe2\x96\x81') - toktypes.append(tt) - tokens.append(vbytes) - scores.append(vscore) - gguf_writer.add_token_list(tokens) - gguf_writer.add_token_scores(scores) - gguf_writer.add_token_types(toktypes) - gguf_writer.add_unk_token_id(0) - gguf_writer.add_bos_token_id(1) - gguf_writer.add_eos_token_id(2) - - def add_tensors(self, gguf_writer): - tensor_map = self.name_map - data = self.data - print(f'* Adding {len(self.model.tensors)} tensor(s)') - for tensor in self.model.tensors: - name = str(tensor.name, 'UTF-8') - mapped_name = tensor_map.get_name(name, try_suffixes = (".weight", ".bias")) - assert mapped_name is not None, f'Bad name {name}' - tempdims = list(tensor.dims[:]) - if len(tempdims) > 1: - temp = tempdims[1] - tempdims[1] = tempdims[0] - tempdims[0] = temp - # print(f'+ {tensor.name} | {mapped_name} {tensor.dims} :: {tempdims}') - gguf_writer.add_tensor(mapped_name, data[tensor.start_offset:tensor.start_offset + tensor.len_bytes], raw_shape = tempdims, raw_dtype = tensor.dtype) - -def handle_metadata(cfg, hp): - import convert - assert cfg.model_metadata_dir.is_dir(), 'Metadata dir is not a directory' - hf_config_path = cfg.model_metadata_dir / "config.json" - orig_config_path = cfg.model_metadata_dir / "params.json" - # We pass a fake model here. "original" mode will check the shapes of some - # tensors if information is missing in the .json file: other than that, the - # model data isn't used so this should be safe (at least for now). - fakemodel = { - 'tok_embeddings.weight': convert.LazyTensor.__new__(convert.LazyTensor), - 'layers.0.feed_forward.w1.weight': convert.LazyTensor.__new__(convert.LazyTensor), - } - fakemodel['tok_embeddings.weight'].shape = [hp.n_vocab] - fakemodel['layers.0.feed_forward.w1.weight'].shape = [hp.n_ff] - if hf_config_path.exists(): - params = convert.Params.loadHFTransformerJson(fakemodel, hf_config_path) - elif orig_config_path.exists(): - params = convert.Params.loadOriginalParamsJson(fakemodel, orig_config_path) - else: - raise ValueError('Unable to load metadata') - vocab = convert.load_vocab(cfg.vocab_dir if cfg.vocab_dir is not None else cfg.model_metadata_dir, cfg.vocabtype) - # FIXME: Respect cfg.vocab_dir? - svocab = gguf.SpecialVocab(cfg.model_metadata_dir) - convert.check_vocab_size(params, vocab) - return (params, vocab, svocab) - -def handle_args(): - parser = argparse.ArgumentParser(description = 'Convert GGMLv3 models to GGUF') - parser.add_argument('--input', '-i', type = Path, required = True, help = 'Input GGMLv3 filename') - parser.add_argument('--output', '-o', type = Path, required = True, help ='Output GGUF filename') - parser.add_argument('--name', help = 'Set model name') - parser.add_argument('--desc', help = 'Set model description') - parser.add_argument('--gqa', type = int, default = 1, help = 'grouped-query attention factor (use 8 for LLaMA2 70B)') - parser.add_argument('--eps', default = '5.0e-06', help = 'RMS norm eps: Use 1e-6 for LLaMA1 and OpenLLaMA, use 1e-5 for LLaMA2') - parser.add_argument('--context-length', '-c', type=int, default = 2048, help = 'Default max context length: LLaMA1 is typically 2048, LLaMA2 is typically 4096') - parser.add_argument('--model-metadata-dir', '-m', type = Path, help ='Load HuggingFace/.pth vocab and metadata from the specified directory') - parser.add_argument("--vocab-dir", type=Path, help="directory containing tokenizer.model, if separate from model file - only meaningful with --model-metadata-dir") - parser.add_argument("--vocabtype", choices=["spm", "bpe"], help="vocab format - only meaningful with --model-metadata-dir and/or --vocab-dir (default: spm)", default="spm") - return parser.parse_args() - -def main(): - cfg = handle_args() - print(f'* Using config: {cfg}') - print('\n=== WARNING === Be aware that this conversion script is best-effort. Use a native GGUF model if possible. === WARNING ===\n') - data = np.memmap(cfg.input, mode = 'r') - model = GGMLV3Model() - print('* Scanning GGML input file') - offset = model.load(data, 0) - print(f'* GGML model hyperparameters: {model.hyperparameters}') - vocab_override = None - params_override = None - special_vocab = None - if cfg.model_metadata_dir is not None: - (params_override, vocab_override, special_vocab) = handle_metadata(cfg, model.hyperparameters) - print('!! Note: When overriding params the --gqa, --eps and --context-length options are ignored.') - print(f'* Overriding params: {params_override}') - print(f'* Overriding vocab: {vocab_override}') - print(f'* Special vocab: {special_vocab}') - else: - print('\n=== WARNING === Special tokens may not be converted correctly. Use --model-metadata-dir if possible === WARNING ===\n') - converter = GGMLToGGUF(model, data, cfg, params_override = params_override, vocab_override = vocab_override, special_vocab = special_vocab) - converter.save() - print(f'* Successful completion. Output saved to: {cfg.output}') - -if __name__ == '__main__': - main()