Skip to content
The Chat Archive
gpt-4-0314· OpenAI WildChat-4.8M (AI2)

I can't modify the C++ code it's not mine I can only edit the export python script. I…

  1. User

    I can't modify the C++ code it's not mine I can only edit the export python script. I want it to split the model in two files consolidated.00.pth consolidated.01.pth with the good layer size.

    Here is how the model is loaded:

    this is the llama_model_function:

    static bool llama_model_load( const std::string & fname, llama_context & lctx, int n_ctx, int n_parts, ggml_type memory_type, bool vocab_only, llama_progress_callback progress_callback, void progress_callback_user_data) { fprintf(stderr, “%s: loading model from ‘%s’ - please wait …\n”, func, fname.c_str());

    lctx.t_start_us = ggml_time_us();

    auto & model = lctx.model; auto & vocab = lctx.vocab;

    auto fin = std::ifstream(fname, std::ios::binary); if (!fin) { fprintf(stderr, “%s: failed to open ‘%s’\n”, func, fname.c_str()); return false; }

    std::vector f_buf(10241024); fin.rdbuf()->pubsetbuf(f_buf.data(), f_buf.size());

    fin.seekg(0, fin.end); const size_t file_size = fin.tellg(); fin.seekg(0);

    // verify magic { uint32_t magic; fin.read((char *) &magic, sizeof(magic)); if (magic == LLAMA_FILE_MAGIC_UNVERSIONED) { fprintf(stderr, “%s: invalid model file ‘%s’ (too old, regenerate your model files or convert them with convert-unversioned-ggml-to-ggml.py!)\n”, func, fname.c_str()); return false; } if (magic != LLAMA_FILE_MAGIC) { return report_bad_magic(fname.c_str(), magic, LLAMA_FILE_MAGIC); }

    uint32_t format_version; fin.read((char *) &format_version, sizeof(format_version));

    if (format_version != LLAMA_FILE_VERSION) { fprintf(stderr, “%s: invalid model file ‘%s’ (unsupported format version %” PRIu32 “, expected %d)\n”, func, fname.c_str(), format_version, LLAMA_FILE_VERSION); return false; } }

    int n_ff = 0;

    // load hparams { auto & hparams = model.hparams;

    fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); //fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd)); fin.read((char ) &hparams.n_mult, sizeof(hparams.n_mult)); fin.read((char ) &hparams.n_head, sizeof(hparams.n_head)); fin.read((char ) &hparams.n_layer, sizeof(hparams.n_layer)); fin.read((char ) &hparams.n_rot, sizeof(hparams.n_rot)); fin.read((char ) &hparams.f16, sizeof(hparams.f16));

    hparams.n_ctx = n_ctx;

    n_ff = ((2(4hparams.n_embd)/3 + hparams.n_mult - 1)/hparams.n_mult)hparams.n_mult;

    if (n_parts < 1) { n_parts = LLAMA_N_PARTS.at(hparams.n_embd); }

    // temp warning to tell the user to use “–n_parts” if (hparams.f16 == 4 && n_parts != 1) { fprintf(stderr, “%s: GPTQ model detected - are you sure n_parts should be %d? we normally expect it to be 1\n”, func, n_parts); fprintf(stderr, “%s: use ‘–n_parts 1’ if necessary\n”, func); }

    if (hparams.n_layer == 32) { model.type = e_model::MODEL_7B; }

    if (hparams.n_layer == 40) { model.type = e_model::MODEL_13B; }

    if (hparams.n_layer == 60) { model.type = e_model::MODEL_30B; }

    if (hparams.n_layer == 80) { model.type = e_model::MODEL_65B; }

    fprintf(stderr, “%s: n_vocab = %d\n”, func, hparams.n_vocab); fprintf(stderr, “%s: n_ctx = %d\n”, func, hparams.n_ctx); fprintf(stderr, “%s: n_embd = %d\n”, func, hparams.n_embd); fprintf(stderr, “%s: n_mult = %d\n”, func, hparams.n_mult); fprintf(stderr, “%s: n_head = %d\n”, func, hparams.n_head); fprintf(stderr, “%s: n_layer = %d\n”, func, hparams.n_layer); fprintf(stderr, “%s: n_rot = %d\n”, func, hparams.n_rot); fprintf(stderr, “%s: f16 = %d\n”, func, hparams.f16); fprintf(stderr, “%s: n_ff = %d\n”, func, n_ff); fprintf(stderr, “%s: n_parts = %d\n”, func, n_parts); fprintf(stderr, “%s: type = %d\n”, func, model.type); }

    // load vocab { std::string word; vocab.id_to_token.resize(model.hparams.n_vocab); std::vector tmp(64);

    for (int i = 0; i < model.hparams.n_vocab; i++) { uint32_t len; fin.read((char ) &len, sizeof(len));

    word.resize(len); if (len > 0) { tmp.resize(len); fin.read(tmp.data(), len); word.assign(tmp.data(), len); } else { word.clear(); }

    float score; fin.read((char ) &score, sizeof(score));

    vocab.token_to_id[word] = i;

    auto &tok_score = vocab.id_to_token[i]; tok_score.tok = word; tok_score.score = score; } }

    if (vocab_only) { return true; }

    // for the big tensors, we have the option to store the data in 16-bit floats or quantized // in order to save memory and also to speed up the computation // wtype is for per-layer weights, while vtype is for other weights ggml_type wtype, vtype; switch (model.hparams.f16) { case 0: wtype = vtype = GGML_TYPE_F32; break; case 1: wtype = vtype = GGML_TYPE_F16; break; case 2: wtype = vtype = GGML_TYPE_Q4_0; break; case 3: wtype = vtype = GGML_TYPE_Q4_1; break; case 4: wtype = GGML_TYPE_Q4_1; vtype = GGML_TYPE_F16; break; default: { fprintf(stderr, “%s: invalid model file ‘%s’ (bad f16 value %d)\n”, func, fname.c_str(), model.hparams.f16); return false; } }

    // map model into memory char mm_addr = NULL; model.mm_addr = mmap_file(fname.c_str(), &model.mm_length); if (model.mm_addr == NULL) { fprintf(stderr, “%s: failed to mmap ‘%s’\n”, func, fname.c_str()); return false; } mm_addr = (char )model.mm_addr; fprintf(stderr, “%s: ggml map size = %6.2f MB\n”, func, model.mm_length/(1024.01024.0));

    auto & ctx = model.ctx;

    size_t ctx_size = 0; { const auto &hparams = model.hparams; const int n_layer = hparams.n_layer; ctx_size += (5 + 10n_layer)256; // object overhead fprintf(stderr, “%s: ggml ctx size = %6.2f KB\n”, func, ctx_size/1024.0); }

    // print memory requirements { const size_t scale = memory_type == GGML_TYPE_F32 ? 2 : 1;

    // this is the total memory required to run the inference const size_t mem_required = ctx_size + model.mm_length + MEM_REQ_SCRATCH0.at(model.type) + MEM_REQ_SCRATCH1.at(model.type) + MEM_REQ_EVAL.at (model.type);

    // this is the memory required by one llama_state const size_t mem_required_state = scaleMEM_REQ_KV_SELF.at(model.type);

    fprintf(stderr, “%s: mem required = %7.2f MB (+ %7.2f MB per state)\n”, func, mem_required / 1024.0 / 1024.0, mem_required_state / 1024.0 / 1024.0); }

    // create the ggml context { lctx.model.buf.resize(ctx_size);

    struct ggml_init_params params = { /.mem_size =/ lctx.model.buf.size(), /.mem_buffer =/ lctx.model.buf.data(), /.no_alloc =/ true, };

    model.ctx = ggml_init(params); if (!model.ctx) { fprintf(stderr, “%s: ggml_init() failed\n”, func); return false; } }

    // prepare memory for the weights { const auto & hparams = model.hparams;

    const int n_embd = hparams.n_embd; const int n_layer = hparams.n_layer; const int n_vocab = hparams.n_vocab;

    model.layers.resize(n_layer);

    model.tok_embeddings = ggml_new_tensor_2d(ctx, vtype, n_embd, n_vocab);

    model.norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); model.output = ggml_new_tensor_2d(ctx, vtype, n_embd, n_vocab);

    // map by name model.tensors[“tok_embeddings.weight”] = model.tok_embeddings;

    model.tensors[“norm.weight”] = model.norm; model.tensors[“output.weight”] = model.output;

    for (int i = 0; i < n_layer; ++i) { auto & layer = model.layers[i];

    layer.attention_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);

    layer.wq = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); layer.wk = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); layer.wv = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); layer.wo = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);

    layer.ffn_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);

    layer.w1 = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff); layer.w2 = ggml_new_tensor_2d(ctx, wtype, n_ff, n_embd); layer.w3 = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);

    // map by name model.tensors[“layers.” + std::to_string(i) + “.attention_norm.weight”] = layer.attention_norm;

    model.tensors[“layers.” + std::to_string(i) + “.attention.wq.weight”] = layer.wq; model.tensors[“layers.” + std::to_string(i) + “.attention.wk.weight”] = layer.wk; model.tensors[“layers.” + std::to_string(i) + “.attention.wv.weight”] = layer.wv; model.tensors[“layers.” + std::to_string(i) + “.attention.wo.weight”] = layer.wo;

    model.tensors[“layers.” + std::to_string(i) + “.ffn_norm.weight”] = layer.ffn_norm;

    model.tensors[“layers.” + std::to_string(i) + “.feed_forward.w1.weight”] = layer.w1; model.tensors[“layers.” + std::to_string(i) + “.feed_forward.w2.weight”] = layer.w2; model.tensors[“layers.” + std::to_string(i) + “.feed_forward.w3.weight”] = layer.w3; } }

    std::vector<uint8_t> tmp;

    if (progress_callback) { progress_callback(0.0, progress_callback_user_data); }

    fprintf(stderr, “%s: loading tensors from ‘%s’\n”, func, fname.c_str());

    // load weights { size_t total_size = 0; model.n_loaded = 0;

    while (true) { int32_t n_dims; int32_t length; int32_t ftype;

    fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims)); fin.read(reinterpret_cast<char *>(&length), sizeof(length)); fin.read(reinterpret_cast<char *>(&ftype), sizeof(ftype));

    if (fin.eof()) { break; }

    int32_t nelements = 1; int32_t ne[2] = { 1, 1 }; for (int i = 0; i < n_dims; ++i) { fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i])); nelements *= ne[i]; }

    std::string name(length, 0); fin.read(&name[0], length);

    if (model.tensors.find(name.data()) == model.tensors.end()) { fprintf(stderr, “%s: unknown tensor ‘%s’ in model file\n”, func, name.data()); return false; }

    auto tensor = model.tensors[name.data()];

    if (ggml_nelements(tensor) != nelements) { fprintf(stderr, “%s: tensor ‘%s’ has wrong size in model file\n”, func, name.data()); return false; } if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) { fprintf(stderr, “%s: tensor ‘%s’ has wrong shape in model file: got [%” PRId64 “, %” PRId64 “], expected [%d, %d]\n”, func, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]); return false; } if (0) { static const char * ftype_str[] = { “f32”, “f16”, “q4_0”, “q4_1”, }; fprintf(stderr, “%24s - [%5d, %5d], type = %6s\n”, name.data(), ne[0], ne[1], ftype_str[ftype]); }

    switch (ftype) { case 0: // f32 case 1: // f16 break; case 2: // q4_0 case 3: // q4_1 assert(ne[0] % 64 == 0); break; default: fprintf(stderr, “%s: unknown ftype %d in model file\n”, func, ftype); return false; };

    // load the tensor data into memory without copying or reading it size_t offset = fin.tellg(); size_t tensor_data_size = ggml_nbytes(tensor); offset = (offset + 31) & -32; tensor->data = mm_addr + offset; fin.seekg(offset + tensor_data_size); total_size += tensor_data_size; model.n_loaded++;

    // progress if (progress_callback) { double current_progress = size_t(fin.tellg()) / double(file_size); progress_callback(current_progress, progress_callback_user_data); } }

    fin.close();

    fprintf(stderr, “%s: model size = %8.2f MB / num tensors = %d\n”, func, total_size/1024.0/1024.0, model.n_loaded); if (model.n_loaded == 0) { fprintf(stderr, “%s: WARN no tensors loaded from model file - assuming empty model for testing\n”, func); } else if (model.n_loaded != (int) model.tensors.size()) { fprintf(stderr, “%s: ERROR not all tensors loaded from model file - expected %zu, got %d\n”, func, model.tensors.size(), model.n_loaded); return false; } }

    // loading time will be recalculate after the first eval, so // we take page faults deferred by mmap() into consideration lctx.t_load_us = ggml_time_us() - lctx.t_start_us;

    if (progress_callback) { progress_callback(1.0, progress_callback_user_data); }

    return true; }

    here is how the model is exported :

    #! /usr/bin/env python

    coding=utf-8

    “”“ Modified from: https://github.com/tloen/alpaca-lora ”“”

    import json import os

    import fire import torch from peft import PeftModel from transformers import LlamaForCausalLM, LlamaTokenizer

    CHECKPOINT_PARAMS = { “7b”: {“dim”: 4096, “multiple_of”: 256, “n_heads”: 32, “n_layers”: 32, “norm_eps”: 1e-06, “vocab_size”: -1}, “13b”: {“dim”: 5120, “multiple_of”: 256, “n_heads”: 40, “n_layers”: 40, “norm_eps”: 1e-06, “vocab_size”: -1}, “30b”: {“dim”: 6656, “multiple_of”: 256, “n_heads”: 52, “n_layers”: 60, “norm_eps”: 1e-06, “vocab_size”: -1}, “65b”: {“dim”: 8192, “multiple_of”: 256, “n_heads”: 64, “n_layers”: 80, “norm_eps”: 1e-06, “vocab_size”: -1}, }

    def main(base_model_name_or_path: str, lora_model_name_or_path: str, output_dir: str, checkpoint_size: str = “7b”):

    Retrieve the model parameters

    params = CHECKPOINT_PARAMS.get(checkpoint_size) if params is None: raise ValueError( f"Cannot find the right model parameters for {checkpoint_size}. Please choose between {list(CHECKPOINT_PARAMS.keys())}.“ )

    tokenizer = LlamaTokenizer.from_pretrained(base_model_name_or_path)

    base_model = LlamaForCausalLM.from_pretrained( base_model_name_or_path, load_in_8bit=False, torch_dtype=torch.float16, device_map={”“: “cpu”}, )

    lora_model = PeftModel.from_pretrained( base_model, lora_model_name_or_path, device_map={”“: “cpu”}, torch_dtype=torch.float16, )

    merge weights

    for layer in lora_model.base_model.model.model.layers: if hasattr(layer.self_attn.q_proj, “merge_weights”): layer.self_attn.q_proj.merge_weights = True if hasattr(layer.self_attn.v_proj, “merge_weights”): layer.self_attn.v_proj.merge_weights = True if hasattr(layer.self_attn.k_proj, “merge_weights”): layer.self_attn.k_proj.merge_weights = True if hasattr(layer.self_attn.o_proj, “merge_weights”): layer.self_attn.o_proj.merge_weights = True if hasattr(layer.mlp.gate_proj, “merge_weights”): layer.mlp.gate_proj.merge_weights = True if hasattr(layer.mlp.down_proj, “merge_weights”): layer.mlp.down_proj.merge_weights = True if hasattr(layer.mlp.up_proj, “merge_weights”): layer.mlp.up_proj.merge_weights = True

    lora_model.train(False)

    lora_model_sd = lora_model.state_dict()

    params = {

    “dim”: 4096,

    “multiple_of”: 256,

    “n_heads”: 32,

    “n_layers”: 32,

    “norm_eps”: 1e-06,

    “vocab_size”: -1,

    }

    n_layers = params[“n_layers”] n_heads = params[“n_heads”] dim = params[“dim”] dims_per_head = dim // n_heads base = 10000.0 inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head))

    def permute(w): return w.view(n_heads, dim // n_heads // 2, 2, dim).transpose(1, 2).reshape(dim, dim)

    def unpermute(w): return w.view(n_heads, 2, dim // n_heads // 2, dim).transpose(1, 2).reshape(dim, dim)

    def translate_state_dict_key(k): k = k.replace(“base_model.model.”, “”) if k == “model.embed_tokens.weight”: return “tok_embeddings.weight” elif k == “model.norm.weight”: return “norm.weight” elif k == “lm_head.weight”: return “output.weight” elif k.startswith(“model.layers.”): layer = k.split(”.“)[2] if k.endswith(”.self_attn.q_proj.weight"): return f"layers.{layer}.attention.wq.weight" elif k.endswith(“.self_attn.k_proj.weight”): return f"layers.{layer}.attention.wk.weight" elif k.endswith(“.self_attn.v_proj.weight”): return f"layers.{layer}.attention.wv.weight" elif k.endswith(“.self_attn.o_proj.weight”): return f"layers.{layer}.attention.wo.weight" elif k.endswith(“.mlp.gate_proj.weight”): return f"layers.{layer}.feed_forward.w1.weight" elif k.endswith(“.mlp.down_proj.weight”): return f"layers.{layer}.feed_forward.w2.weight" elif k.endswith(“.mlp.up_proj.weight”): return f"layers.{layer}.feed_forward.w3.weight" elif k.endswith(“.input_layernorm.weight”): return f"layers.{layer}.attention_norm.weight" elif k.endswith(“.post_attention_layernorm.weight”): return f"layers.{layer}.ffn_norm.weight" elif k.endswith(“rotary_emb.inv_freq”) or “lora” in k: return None else: print(layer, k) raise NotImplementedError else: print(k) raise NotImplementedError

    new_state_dict = {} for k, v in lora_model_sd.items(): new_k = translate_state_dict_key(k) if new_k is not None: if “wq” in new_k or “wk” in new_k: new_state_dict[new_k] = unpermute(v) else: new_state_dict[new_k] = v

    os.makedirs(output_dir, exist_ok=True)

    Split the tensors based on layer index

    n_layers_actual = len([k for k in new_state_dict.keys() if ".attention.wq.weight" in k])
    part1_keys = [k for k in new_state_dict.keys() if not k.startswith("layers.") or int(k.split(".")[1]) < (n_layers_actual // 2)]
    part2_keys = [k for k in new_state_dict.keys() if k not in part1_keys]
    
    state_dict_part1 = {k: new_state_dict[k] for k in part1_keys}
    state_dict_part2 = {k: new_state_dict[k] for k in part2_keys}
    
    torch.save(state_dict_part1, output_dir + "/consolidated.00.pth")
    torch.save(state_dict_part2, output_dir + "/consolidated.01.pth")
    with open(output_dir + "/params.json", "w") as f:
        json.dump(params, f)
    

    if name == “main”: fire.Fire(main)

    Here is the problem I have when i run the inference:

    ./main -m ./models/13B/ggml-model-f16.bin -n 5000 --repeat_penalty 1.0 --color -i -r “User:” -f prompts/chat-with-bob.txt -t 32 main: seed = 1681035697 llama_model_load: loading model from ‘./models/13B/ggml-model-f16.bin’ - please wait … llama_model_load: n_vocab = 32000 llama_model_load: n_ctx = 512 llama_model_load: n_embd = 5120 llama_model_load: n_mult = 256 llama_model_load: n_head = 40 llama_model_load: n_layer = 40 llama_model_load: n_rot = 128 llama_model_load: f16 = 1 llama_model_load: n_ff = 13824 llama_model_load: n_parts = 2 llama_model_load: type = 2 llama_model_load: ggml map size = 25138.72 MB llama_model_load: ggml ctx size = 101.25 KB llama_model_load: mem required = 27186.82 MB (+ 1608.00 MB per state) llama_model_load: loading tensors from ‘./models/13B/ggml-model-f16.bin’ llama_model_load: tensor ‘layers.20.attention.wq.weight’ has wrong size in model file llama_init_from_file: failed to load model main: error: failed to load model ‘./models/13B/ggml-model-f16.bin’

  2. Assistant

    It appears that the issue is occurring during the model loading process in the llama_model_load function. The error message suggests that there is a mismatch in the size of the tensor layers.20.attention.wq.weight in the model file.

    To resolve this issue, you can first verify if the exported model tensors have the correct size by printing the shapes of tensors during the export process. Modify the translate_state_dict_key function in the model export script as follows:

    def translate_state_dict_key(k):
        # Add the following line to print the shape of each tensor during export
        print(f"Exporting tensor: {k}, Shape: {lora_model_sd[k].shape}")
    
        k = k.replace("base_model.model.", "")
        # Rest of the function remains unchanged
    

    Then, export the model again and check the console output to see if there are any mismatches in the size of the tensors. This should help you identify if the issue stems from the export process or the loading process.

    Next, you can modify the llama_model_load function to print the expected and actual sizes of the tensors during the loading process. Add a print statement before the error message like this:

    if (ggml_nelements(tensor) != nelements) {
        fprintf(stderr, "%s: tensor ‘%s’ has wrong size in model file\n", func, name.data());
    
        // Add the following line
        fprintf(stderr, "Expected size: %d, Actual size: %d\n", nelements, ggml_nelements(tensor));
        return false;
    }
    

    Now, recompile and run the model loading process again to see if there are any mismatches in the size of the tensors. This information should help you identify the reason behind the size mismatches and decide the next steps to fix the issue.