gpt-4-0314· OpenAI WildChat-4.8M (AI2)

Instead of having a model saved in one consolidated.00.pth i would like the model being…

User

Instead of having a model saved in one consolidated.00.pth i would like the model being splited in 2 files:

#! /usr/bin/env python

coding=utf-8

""" Modified from: https://github.com/tloen/alpaca-lora """

import json import os

import fire import torch from peft import PeftModel from transformers import LlamaForCausalLM, LlamaTokenizer

CHECKPOINT_PARAMS = { "7b": {"dim": 4096, "multiple_of": 256, "n_heads": 32, "n_layers": 32, "norm_eps": 1e-06, "vocab_size": -1}, "13b": {"dim": 5120, "multiple_of": 256, "n_heads": 40, "n_layers": 40, "norm_eps": 1e-06, "vocab_size": -1}, "30b": {"dim": 6656, "multiple_of": 256, "n_heads": 52, "n_layers": 60, "norm_eps": 1e-06, "vocab_size": -1}, "65b": {"dim": 8192, "multiple_of": 256, "n_heads": 64, "n_layers": 80, "norm_eps": 1e-06, "vocab_size": -1}, }

def main(base_model_name_or_path: str, lora_model_name_or_path: str, output_dir: str, checkpoint_size: str = "7b"):

# Retrieve the model parameters
params = CHECKPOINT_PARAMS.get(checkpoint_size)
if params is None:
    raise ValueError(
        f"Cannot find the right model parameters for {checkpoint_size}. Please choose between {list(CHECKPOINT_PARAMS.keys())}."
    )

# tokenizer = LlamaTokenizer.from_pretrained(base_model_name_or_path)

base_model = LlamaForCausalLM.from_pretrained(
    base_model_name_or_path,
    load_in_8bit=False,
    torch_dtype=torch.float16,
    device_map={"": "cpu"},
)

lora_model = PeftModel.from_pretrained(
    base_model,
    lora_model_name_or_path,
    device_map={"": "cpu"},
    torch_dtype=torch.float16,
)

# merge weights
for layer in lora_model.base_model.model.model.layers:
    if hasattr(layer.self_attn.q_proj, "merge_weights"):
        layer.self_attn.q_proj.merge_weights = True
    if hasattr(layer.self_attn.v_proj, "merge_weights"):
        layer.self_attn.v_proj.merge_weights = True
    if hasattr(layer.self_attn.k_proj, "merge_weights"):
        layer.self_attn.k_proj.merge_weights = True
    if hasattr(layer.self_attn.o_proj, "merge_weights"):
        layer.self_attn.o_proj.merge_weights = True
    if hasattr(layer.mlp.gate_proj, "merge_weights"):
        layer.mlp.gate_proj.merge_weights = True
    if hasattr(layer.mlp.down_proj, "merge_weights"):
        layer.mlp.down_proj.merge_weights = True
    if hasattr(layer.mlp.up_proj, "merge_weights"):
        layer.mlp.up_proj.merge_weights = True

lora_model.train(False)

lora_model_sd = lora_model.state_dict()

# params = {
#     "dim": 4096,
#     "multiple_of": 256,
#     "n_heads": 32,
#     "n_layers": 32,
#     "norm_eps": 1e-06,
#     "vocab_size": -1,
# }
n_layers = params["n_layers"]
n_heads = params["n_heads"]
dim = params["dim"]
dims_per_head = dim // n_heads
base = 10000.0
inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head))

def permute(w):
    return w.view(n_heads, dim // n_heads // 2, 2, dim).transpose(1, 2).reshape(dim, dim)

def unpermute(w):
    return w.view(n_heads, 2, dim // n_heads // 2, dim).transpose(1, 2).reshape(dim, dim)

def translate_state_dict_key(k):
    k = k.replace("base_model.model.", "")
    if k == "model.embed_tokens.weight":
        return "tok_embeddings.weight"
    elif k == "model.norm.weight":
        return "norm.weight"
    elif k == "lm_head.weight":
        return "output.weight"
    elif k.startswith("model.layers."):
        layer = k.split(".")[2]
        if k.endswith(".self_attn.q_proj.weight"):
            return f"layers.{layer}.attention.wq.weight"
        elif k.endswith(".self_attn.k_proj.weight"):
            return f"layers.{layer}.attention.wk.weight"
        elif k.endswith(".self_attn.v_proj.weight"):
            return f"layers.{layer}.attention.wv.weight"
        elif k.endswith(".self_attn.o_proj.weight"):
            return f"layers.{layer}.attention.wo.weight"
        elif k.endswith(".mlp.gate_proj.weight"):
            return f"layers.{layer}.feed_forward.w1.weight"
        elif k.endswith(".mlp.down_proj.weight"):
            return f"layers.{layer}.feed_forward.w2.weight"
        elif k.endswith(".mlp.up_proj.weight"):
            return f"layers.{layer}.feed_forward.w3.weight"
        elif k.endswith(".input_layernorm.weight"):
            return f"layers.{layer}.attention_norm.weight"
        elif k.endswith(".post_attention_layernorm.weight"):
            return f"layers.{layer}.ffn_norm.weight"
        elif k.endswith("rotary_emb.inv_freq") or "lora" in k:
            return None
        else:
            print(layer, k)
            raise NotImplementedError
    else:
        print(k)
        raise NotImplementedError

new_state_dict = {}
for k, v in lora_model_sd.items():
    new_k = translate_state_dict_key(k)
    if new_k is not None:
        if "wq" in new_k or "wk" in new_k:
            new_state_dict[new_k] = unpermute(v)
        else:
            new_state_dict[new_k] = v

os.makedirs(output_dir, exist_ok=True)

torch.save(new_state_dict, output_dir + "/consolidated.00.pth")

with open(output_dir + "/params.json", "w") as f:
    json.dump(params, f)

if name == "main": fire.Fire(main)

Assistant2023-04-09 09:09 UTC

You can modify the provided script to save the model in two separate files. For example, you can split the model into its embedding layer and the remaining layers. Here's an example of how to do it:

import json
import os

import fire
import torch
from peft import PeftModel
from transformers import LlamaForCausalLM, LlamaTokenizer

CHECKPOINT_PARAMS = {
    "7b": {"dim": 4096, "multiple_of": 256, "n_heads": 32, "n_layers": 32, "norm_eps": 1e-06, "vocab_size": -1},
    # ... (remaining size configurations)
}

def main(base_model_name_or_path: str, lora_model_name_or_path: str, output_dir: str, checkpoint_size: str = "7b"):
    # ... (initialize models and set merge_weights attributes)
  
    lora_model.train(False)
    lora_model_sd = lora_model.state_dict()

    # ... (define params, n_layers, n_heads, dim, dims_per_head, base, inv_freq, permute, unpermute, and translate_state_dict_key)
    
    embedding_state_dict = {}
    remaining_state_dict = {}
    
    for k, v in lora_model_sd.items():
        new_k = translate_state_dict_key(k)
        if new_k is not None:
            if "tok_embeddings" in new_k:
                embedding_state_dict[new_k] = v
            else:
                if "wq" in new_k or "wk" in new_k:
                    remaining_state_dict[new_k] = unpermute(v)
                else:
                    remaining_state_dict[new_k] = v
    
    os.makedirs(output_dir, exist_ok=True)
    
    torch.save(embedding_state_dict, output_dir + "/embedding.00.pth")
    torch.save(remaining_state_dict, output_dir + "/remaining.00.pth")

    with open(output_dir + "/params.json", "w") as f:
        json.dump(params, f)
        
if __name__ == "__main__":
    fire.Fire(main)

This script will save the embedding layer in a file named embedding.00.pth and the remaining layers in a file named remaining.00.pth. Note that you will need to adjust your model loading code to account for the split state_dicts when using the saved model.

User
No i mean splited in two files like consolidated.00.pth consolidated.01.pth

Assistant2023-04-09 09:13 UTC

Ah, I see. In that case, you can simply split the state dictionary into two parts and save them in separate files. Here's an example of how to do it:

import json
import os

import fire
import torch
from peft import PeftModel
from transformers import LlamaForCausalLM, LlamaTokenizer

CHECKPOINT_PARAMS = {
    "7b": {"dim": 4096, "multiple_of": 256, "n_heads": 32, "n_layers": 32, "norm_eps": 1e-06, "vocab_size": -1},
    # ... (remaining size configurations)
}

def main(base_model_name_or_path: str, lora_model_name_or_path: str, output_dir: str, checkpoint_size: str = "7b"):
    # ... (initialize models and set merge_weights attributes)
  
    lora_model.train(False)
    lora_model_sd = lora_model.state_dict()

    # ... (define params, n_layers, n_heads, dim, dims_per_head, base, inv_freq, permute, unpermute, and translate_state_dict_key)
    
    new_state_dict = {}
    for k, v in lora_model_sd.items():
        new_k = translate_state_dict_key(k)
        if new_k is not None:
            if "wq" in new_k or "wk" in new_k:
                new_state_dict[new_k] = unpermute(v)
            else:
                new_state_dict[new_k] = v
    
    os.makedirs(output_dir, exist_ok=True)

    # Split the state_dict into two equal parts
    state_dict_items = list(new_state_dict.items())
    half_len = len(state_dict_items) // 2

    state_dict_part1 = dict(state_dict_items[:half_len])
    state_dict_part2 = dict(state_dict_items[half_len:])

    torch.save(state_dict_part1, output_dir + "/consolidated.00.pth")
    torch.save(state_dict_part2, output_dir + "/consolidated.01.pth")

    with open(output_dir + "/params.json", "w") as f:
        json.dump(params, f)
        
if __name__ == "__main__":
    fire.Fire(main)

This script will save the first half of the new_state_dict in a file named consolidated.00.pth and the remaining in a file named consolidated.01.pth. Note that you will need to adjust your model loading code to account for the split state_dicts when using the saved model.

User
it worked correctly i Have the number of consolidated files for the inference to run but I have this problem:

./main -m ./models/13B/ggml-model-q4_0.bin -n 5000 --repeat_penalty 1.0 --color -i -r "User:" -f prompts/chat-with-bob.txt -t 32 main: seed = 1681033517 llama_model_load: loading model from './models/13B/ggml-model-q4_0.bin' - please wait ... llama_model_load: n_vocab = 32000 llama_model_load: n_ctx = 512 llama_model_load: n_embd = 5120 llama_model_load: n_mult = 256 llama_model_load: n_head = 40 llama_model_load: n_layer = 40 llama_model_load: n_rot = 128 llama_model_load: f16 = 2 llama_model_load: n_ff = 13824 llama_model_load: n_parts = 2 llama_model_load: type = 2 llama_model_load: ggml map size = 7759.04 MB llama_model_load: ggml ctx size = 101.25 KB llama_model_load: mem required = 9807.14 MB (+ 1608.00 MB per state) llama_model_load: loading tensors from './models/13B/ggml-model-q4_0.bin' llama_model_load: tensor 'layers.20.attention.wq.weight' has wrong size in model file llama_init_from_file: failed to load model main: error: failed to load model './models/13B/ggml-model-q4_0.bin'
Assistant2023-04-09 09:47 UTC
It seems that the error occurs when trying to load the checkpoint model files, and the issue is related to a mismatch in the size of the loaded tensor layers.20.attention.wq.weight.

To provide a more precise solution, I would need more details about llama_model_load function and how it loads the tensors from the consolidated files.

However, I can give a general suggestion for debugging the problem. Try the following steps to find the cause of the issue and to fix it:
1. Verify that the correct model files are loaded. It may sometimes happen that incorrect files are inadvertently passed to the model loading function, especially when working with multiple models.
2. Verify the size of the tensor layers.20.attention.wq.weight in the checkpoint files. You can do this by loading the tensor from the consolidated.00.pth and consolidated.01.pth files in Python.
3. Compare the size of the tensor during the model saving and model loading phases. Ensure the sizes match or adjust the processing steps in either the model saving or model loading code accordingly.
If these steps do not help to solve the problem or you need more assistance, please provide more information about the llama_model_load function and the steps it takes to load model tensors from the consolidated files.

User

this is the llama_model_function:

static bool llama_model_load( const std::string & fname, llama_context & lctx, int n_ctx, int n_parts, ggml_type memory_type, bool vocab_only, llama_progress_callback progress_callback, void *progress_callback_user_data) { fprintf(stderr, "%s: loading model from '%s' - please wait ...\n", func, fname.c_str());

lctx.t_start_us = ggml_time_us();

auto & model = lctx.model;
auto & vocab = lctx.vocab;

auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
    fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
    return false;
}

std::vector<char> f_buf(1024*1024);
fin.rdbuf()->pubsetbuf(f_buf.data(), f_buf.size());

fin.seekg(0, fin.end);
const size_t file_size = fin.tellg();
fin.seekg(0);

// verify magic
{
    uint32_t magic;
    fin.read((char *) &magic, sizeof(magic));
    if (magic == LLAMA_FILE_MAGIC_UNVERSIONED) {
        fprintf(stderr, "%s: invalid model file '%s' (too old, regenerate your model files or convert them with convert-unversioned-ggml-to-ggml.py!)\n",
                __func__, fname.c_str());
        return false;
    }
    if (magic != LLAMA_FILE_MAGIC) {
        return report_bad_magic(fname.c_str(), magic, LLAMA_FILE_MAGIC);
    }

    uint32_t format_version;
    fin.read((char *) &format_version, sizeof(format_version));

    if (format_version != LLAMA_FILE_VERSION) {
        fprintf(stderr, "%s: invalid model file '%s' (unsupported format version %" PRIu32 ", expected %d)\n",
                __func__, fname.c_str(), format_version, LLAMA_FILE_VERSION);
        return false;
    }
}

int n_ff = 0;

// load hparams
{
    auto & hparams = model.hparams;

    fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
    //fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
    fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
    fin.read((char *) &hparams.n_mult,  sizeof(hparams.n_mult));
    fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
    fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
    fin.read((char *) &hparams.n_rot,   sizeof(hparams.n_rot));
    fin.read((char *) &hparams.f16,     sizeof(hparams.f16));

    hparams.n_ctx = n_ctx;

    n_ff = ((2*(4*hparams.n_embd)/3 + hparams.n_mult - 1)/hparams.n_mult)*hparams.n_mult;

    if (n_parts < 1) {
        n_parts = LLAMA_N_PARTS.at(hparams.n_embd);
    }

    // temp warning to tell the user to use "--n_parts"
    if (hparams.f16 == 4 && n_parts != 1) {
        fprintf(stderr, "%s: GPTQ model detected - are you sure n_parts should be %d? we normally expect it to be 1\n", __func__, n_parts);
        fprintf(stderr, "%s: use '--n_parts 1' if necessary\n", __func__);
    }

    if (hparams.n_layer == 32) {
        model.type = e_model::MODEL_7B;
    }

    if (hparams.n_layer == 40) {
        model.type = e_model::MODEL_13B;
    }

    if (hparams.n_layer == 60) {
        model.type = e_model::MODEL_30B;
    }

    if (hparams.n_layer == 80) {
        model.type = e_model::MODEL_65B;
    }

    fprintf(stderr, "%s: n_vocab = %d\n", __func__, hparams.n_vocab);
    fprintf(stderr, "%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
    fprintf(stderr, "%s: n_embd  = %d\n", __func__, hparams.n_embd);
    fprintf(stderr, "%s: n_mult  = %d\n", __func__, hparams.n_mult);
    fprintf(stderr, "%s: n_head  = %d\n", __func__, hparams.n_head);
    fprintf(stderr, "%s: n_layer = %d\n", __func__, hparams.n_layer);
    fprintf(stderr, "%s: n_rot   = %d\n", __func__, hparams.n_rot);
    fprintf(stderr, "%s: f16     = %d\n", __func__, hparams.f16);
    fprintf(stderr, "%s: n_ff    = %d\n", __func__, n_ff);
    fprintf(stderr, "%s: n_parts = %d\n", __func__, n_parts);
    fprintf(stderr, "%s: type    = %d\n", __func__, model.type);
}

// load vocab
{
    std::string word;
    vocab.id_to_token.resize(model.hparams.n_vocab);
    std::vector<char> tmp(64);

    for (int i = 0; i < model.hparams.n_vocab; i++) {
        uint32_t len;
        fin.read((char *) &len, sizeof(len));

        word.resize(len);
        if (len > 0) {
            tmp.resize(len);
            fin.read(tmp.data(), len);
            word.assign(tmp.data(), len);
        } else {
            word.clear();
        }

        float score;
        fin.read((char *) &score, sizeof(score));

        vocab.token_to_id[word] = i;

        auto &tok_score = vocab.id_to_token[i];
        tok_score.tok = word;
        tok_score.score = score;
    }
}

if (vocab_only) {
    return true;
}

// for the big tensors, we have the option to store the data in 16-bit floats or quantized
// in order to save memory and also to speed up the computation
// wtype is for per-layer weights, while vtype is for other weights
ggml_type wtype, vtype;
switch (model.hparams.f16) {
    case 0: wtype = vtype = GGML_TYPE_F32;  break;
    case 1: wtype = vtype = GGML_TYPE_F16;  break;
    case 2: wtype = vtype = GGML_TYPE_Q4_0; break;
    case 3: wtype = vtype = GGML_TYPE_Q4_1; break;
    case 4: wtype = GGML_TYPE_Q4_1; vtype = GGML_TYPE_F16; break;
    default:
            {
                fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
                        __func__, fname.c_str(), model.hparams.f16);
                return false;
            }
}

// map model into memory
char *mm_addr = NULL;
model.mm_addr = mmap_file(fname.c_str(), &model.mm_length);
if (model.mm_addr == NULL) {
    fprintf(stderr, "%s: failed to mmap '%s'\n", __func__, fname.c_str());
    return false;
}
mm_addr = (char *)model.mm_addr;
fprintf(stderr, "%s: ggml map size = %6.2f MB\n", __func__, model.mm_length/(1024.0*1024.0));

auto & ctx = model.ctx;

size_t ctx_size = 0;
{
    const auto &hparams = model.hparams;
    const int n_layer = hparams.n_layer;
    ctx_size += (5 + 10*n_layer)*256; // object overhead
    fprintf(stderr, "%s: ggml ctx size = %6.2f KB\n", __func__, ctx_size/1024.0);
}

// print memory requirements
{
    const size_t scale = memory_type == GGML_TYPE_F32 ? 2 : 1;

    // this is the total memory required to run the inference
    const size_t mem_required =
        ctx_size +
        model.mm_length +
        MEM_REQ_SCRATCH0.at(model.type) +
        MEM_REQ_SCRATCH1.at(model.type) +
        MEM_REQ_EVAL.at    (model.type);

    // this is the memory required by one llama_state
    const size_t mem_required_state =
        scale*MEM_REQ_KV_SELF.at(model.type);

    fprintf(stderr, "%s: mem required  = %7.2f MB (+ %7.2f MB per state)\n", __func__,
            mem_required / 1024.0 / 1024.0, mem_required_state / 1024.0 / 1024.0);
}

// create the ggml context
{
    lctx.model.buf.resize(ctx_size);

    struct ggml_init_params params = {
        /*.mem_size   =*/ lctx.model.buf.size(),
        /*.mem_buffer =*/ lctx.model.buf.data(),
        /*.no_alloc   =*/ true,
    };

    model.ctx = ggml_init(params);
    if (!model.ctx) {
        fprintf(stderr, "%s: ggml_init() failed\n", __func__);
        return false;
    }
}

// prepare memory for the weights
{
    const auto & hparams = model.hparams;

    const int n_embd  = hparams.n_embd;
    const int n_layer = hparams.n_layer;
    const int n_vocab = hparams.n_vocab;

    model.layers.resize(n_layer);

    model.tok_embeddings = ggml_new_tensor_2d(ctx, vtype, n_embd, n_vocab);

    model.norm   = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
    model.output = ggml_new_tensor_2d(ctx, vtype,         n_embd, n_vocab);

    // map by name
    model.tensors["tok_embeddings.weight"] = model.tok_embeddings;

    model.tensors["norm.weight"]   = model.norm;
    model.tensors["output.weight"] = model.output;

    for (int i = 0; i < n_layer; ++i) {
        auto & layer = model.layers[i];

        layer.attention_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);

        layer.wq = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
        layer.wk = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
        layer.wv = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
        layer.wo = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);

        layer.ffn_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);

        layer.w1 = ggml_new_tensor_2d(ctx, wtype, n_embd,   n_ff);
        layer.w2 = ggml_new_tensor_2d(ctx, wtype,   n_ff, n_embd);
        layer.w3 = ggml_new_tensor_2d(ctx, wtype, n_embd,   n_ff);

        // map by name
        model.tensors["layers." + std::to_string(i) + ".attention_norm.weight"] = layer.attention_norm;

        model.tensors["layers." + std::to_string(i) + ".attention.wq.weight"] = layer.wq;
        model.tensors["layers." + std::to_string(i) + ".attention.wk.weight"] = layer.wk;
        model.tensors["layers." + std::to_string(i) + ".attention.wv.weight"] = layer.wv;
        model.tensors["layers." + std::to_string(i) + ".attention.wo.weight"] = layer.wo;

        model.tensors["layers." + std::to_string(i) + ".ffn_norm.weight"] = layer.ffn_norm;

        model.tensors["layers." + std::to_string(i) + ".feed_forward.w1.weight"] = layer.w1;
        model.tensors["layers." + std::to_string(i) + ".feed_forward.w2.weight"] = layer.w2;
        model.tensors["layers." + std::to_string(i) + ".feed_forward.w3.weight"] = layer.w3;
    }
}

std::vector<uint8_t> tmp;

if (progress_callback) {
    progress_callback(0.0, progress_callback_user_data);
}

fprintf(stderr, "%s: loading tensors from '%s'\n", __func__, fname.c_str());

// load weights
{
    size_t total_size = 0;
    model.n_loaded = 0;

    while (true) {
        int32_t n_dims;
        int32_t length;
        int32_t ftype;

        fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
        fin.read(reinterpret_cast<char *>(&length), sizeof(length));
        fin.read(reinterpret_cast<char *>(&ftype),  sizeof(ftype));

        if (fin.eof()) {
            break;
        }

        int32_t nelements = 1;
        int32_t ne[2] = { 1, 1 };
        for (int i = 0; i < n_dims; ++i) {
            fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
            nelements *= ne[i];
        }

        std::string name(length, 0);
        fin.read(&name[0], length);

        if (model.tensors.find(name.data()) == model.tensors.end()) {
            fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
            return false;
        }

        auto tensor = model.tensors[name.data()];

        if (ggml_nelements(tensor) != nelements) {
            fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
            return false;
        }
        if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
            fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
                    __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
            return false;
        }
        if (0) {
            static const char * ftype_str[] = { "f32", "f16", "q4_0", "q4_1", };
            fprintf(stderr, "%24s - [%5d, %5d], type = %6s\n", name.data(), ne[0], ne[1], ftype_str[ftype]);
        }

        switch (ftype) {
            case 0:  // f32
            case 1:  // f16
                break;
            case 2:  // q4_0
            case 3:  // q4_1
                assert(ne[0] % 64 == 0);
                break;
            default:
                fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype);
                return false;
        };

        // load the tensor data into memory without copying or reading it
        size_t offset = fin.tellg();
        size_t tensor_data_size = ggml_nbytes(tensor);
        offset = (offset + 31) & -32;
        tensor->data = mm_addr + offset;
        fin.seekg(offset + tensor_data_size);
        total_size += tensor_data_size;
        model.n_loaded++;

        // progress
        if (progress_callback) {
            double current_progress = size_t(fin.tellg()) / double(file_size);
            progress_callback(current_progress, progress_callback_user_data);
        }
    }

    fin.close();

    fprintf(stderr, "%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, model.n_loaded);
    if (model.n_loaded == 0) {
        fprintf(stderr, "%s: WARN no tensors loaded from model file - assuming empty model for testing\n", __func__);
    } else if (model.n_loaded != (int) model.tensors.size()) {
        fprintf(stderr, "%s: ERROR not all tensors loaded from model file - expected %zu, got %d\n", __func__, model.tensors.size(), model.n_loaded);
        return false;
    }
}

// loading time will be recalculate after the first eval, so
// we take page faults deferred by mmap() into consideration
lctx.t_load_us = ggml_time_us() - lctx.t_start_us;

if (progress_callback) {
    progress_callback(1.0, progress_callback_user_data);
}

return true;

}

Assistant2023-04-09 09:52 UTC

Since the llama_model_load function expects the tensors to be in a single file, we should adjust the way tensors are saved in the original Python script. Instead of splitting the tensors evenly between two files, we can save the tensors based on their layer indices so that the loading function can read them seamlessly.

Here's a modified version of the script that saves the tensors properly for llama_model_load function:

import json
import os

import fire
import torch
from peft import PeftModel
from transformers import LlamaForCausalLM, LlamaTokenizer

CHECKPOINT_PARAMS = {
    "7b": {"dim": 4096, "multiple_of": 256, "n_heads": 32, "n_layers": 32, "norm_eps": 1e-06, "vocab_size": -1},
    # ... (remaining size configurations)
}

def main(base_model_name_or_path: str, lora_model_name_or_path: str, output_dir: str, checkpoint_size: str = "7b"):
    # ... (initialize models and set merge_weights attributes)
  
    lora_model.train(False)
    lora_model_sd = lora_model.state_dict()

    # ... (define params, n_layers, n_heads, dim, dims_per_head, base, inv_freq, permute, unpermute, and translate_state_dict_key)
    
    new_state_dict = {}
    for k, v in lora_model_sd.items():
        new_k = translate_state_dict_key(k)
        if new_k is not None:
            if "wq" in new_k or "wk" in new_k:
                new_state_dict[new_k] = unpermute(v)
            else:
                new_state_dict[new_k] = v
    
    os.makedirs(output_dir, exist_ok=True)

    # Split the tensors based on layer index
    part1_keys = [k for k in new_state_dict.keys() if not k.startswith("layers.") or int(k.split(".")[1]) < n_layers // 2]
    part2_keys = [k for k in new_state_dict.keys() if k not in part1_keys]

    state_dict_part1 = {k: new_state_dict[k] for k in part1_keys}
    state_dict_part2 = {k: new_state_dict[k] for k in part2_keys}

    torch.save(state_dict_part1, output_dir + "/consolidated.00.pth")
    torch.save(state_dict_part2, output_dir + "/consolidated.01.pth")

    with open(output_dir + "/params.json", "w") as f:
        json.dump(params, f)
        
if __name__ == "__main__":
    fire.Fire(main)

The script modifies the way tensors are split between the two consolidated.00.pth and consolidated.01.pth files. This should resolve the issue when loading the tensors during inference.

Let me know if you need further assistance.

User
there still have problems in layer 20:

./main -m ./models/13B/ggml-model-f16.bin -n 5000 --repeat_penalty 1.0 --color -i -r "User:" -f prompts/chat-with-bob.txt -t 32 main: seed = 1681035697 llama_model_load: loading model from './models/13B/ggml-model-f16.bin' - please wait ... llama_model_load: n_vocab = 32000 llama_model_load: n_ctx = 512 llama_model_load: n_embd = 5120 llama_model_load: n_mult = 256 llama_model_load: n_head = 40 llama_model_load: n_layer = 40 llama_model_load: n_rot = 128 llama_model_load: f16 = 1 llama_model_load: n_ff = 13824 llama_model_load: n_parts = 2 llama_model_load: type = 2 llama_model_load: ggml map size = 25138.72 MB llama_model_load: ggml ctx size = 101.25 KB llama_model_load: mem required = 27186.82 MB (+ 1608.00 MB per state) llama_model_load: loading tensors from './models/13B/ggml-model-f16.bin' llama_model_load: tensor 'layers.20.attention.wq.weight' has wrong size in model file llama_init_from_file: failed to load model main: error: failed to load model './models/13B/ggml-model-f16.bin'

Assistant2023-04-09 10:23 UTC

I apologize for the confusion. It appears that the tensor splitting modification was not correct. I noticed one mistake in my code that should be fixed. The problem is with determining if the key starts with "layers." it should be numerial value.

Here is a corrected version:

import json
import os

import fire
import torch
from peft import PeftModel
from transformers import LlamaForCausalLM, LlamaTokenizer

CHECKPOINT_PARAMS = {
    "7b": {"dim": 4096, "multiple_of": 256, "n_heads": 32, "n_layers": 32, "norm_eps": 1e-06, "vocab_size": -1},
    # ... (remaining size configurations)
}

def main(base_model_name_or_path: str, lora_model_name_or_path: str, output_dir: str, checkpoint_size: str = "7b"):
    # ... (initialize models and set merge_weights attributes)
  
    lora_model.train(False)
    lora_model_sd = lora_model.state_dict()

    # ... (define params, n_layers, n_heads, dim, dims_per_head, base, inv_freq, permute, unpermute, and translate_state_dict_key)
    
    new_state_dict = {}
    for k, v in lora_model_sd.items():
        new_k = translate_state_dict_key(k)
        if new_k is not None:
            if "wq" in new_k or "wk" in new_k:
                new_state_dict[new_k] = unpermute(v)
            else:
                new_state