I can only change the export python script, i need it to split model files in two files…

I can only change the export python script, i need it to split model files in two files consolidated.00.pth consolidated.01.pth with the correct layers size:

this is the llama_model_function:

static bool llama_model_load( const std::string & fname, llama_context & lctx, int n_ctx, int n_parts, ggml_type memory_type, bool vocab_only, llama_progress_callback progress_callback, void progress_callback_user_data) { fprintf(stderr, “%s: loading model from ‘%s’ - please wait …\n”, func, fname.c_str());

lctx.t_start_us = ggml_time_us();

auto & model = lctx.model; auto & vocab = lctx.vocab;

auto fin = std::ifstream(fname, std::ios::binary); if (!fin) { fprintf(stderr, “%s: failed to open ‘%s’\n”, func, fname.c_str()); return false; }

std::vector f_buf(10241024); fin.rdbuf()->pubsetbuf(f_buf.data(), f_buf.size());

fin.seekg(0, fin.end); const size_t file_size = fin.tellg(); fin.seekg(0);

// verify magic { uint32_t magic; fin.read((char *) &magic, sizeof(magic)); if (magic == LLAMA_FILE_MAGIC_UNVERSIONED) { fprintf(stderr, “%s: invalid model file ‘%s’ (too old, regenerate your model files or convert them with convert-unversioned-ggml-to-ggml.py!)\n”, func, fname.c_str()); return false; } if (magic != LLAMA_FILE_MAGIC) { return report_bad_magic(fname.c_str(), magic, LLAMA_FILE_MAGIC); }

uint32_t format_version; fin.read((char *) &format_version, sizeof(format_version));

if (format_version != LLAMA_FILE_VERSION) { fprintf(stderr, “%s: invalid model file ‘%s’ (unsupported format version %” PRIu32 “, expected %d)\n”, func, fname.c_str(), format_version, LLAMA_FILE_VERSION); return false; } }

int n_ff = 0;

// load hparams { auto & hparams = model.hparams;

fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); //fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd)); fin.read((char ) &hparams.n_mult, sizeof(hparams.n_mult)); fin.read((char ) &hparams.n_head, sizeof(hparams.n_head)); fin.read((char ) &hparams.n_layer, sizeof(hparams.n_layer)); fin.read((char ) &hparams.n_rot, sizeof(hparams.n_rot)); fin.read((char ) &hparams.f16, sizeof(hparams.f16));

hparams.n_ctx = n_ctx;

n_ff = ((2(4hparams.n_embd)/3 + hparams.n_mult - 1)/hparams.n_mult)hparams.n_mult;

if (n_parts < 1) { n_parts = LLAMA_N_PARTS.at(hparams.n_embd); }

// temp warning to tell the user to use “–n_parts” if (hparams.f16 == 4 && n_parts != 1) { fprintf(stderr, “%s: GPTQ model detected - are you sure n_parts should be %d? we normally expect it to be 1\n”, func, n_parts); fprintf(stderr, “%s: use ‘–n_parts 1’ if necessary\n”, func); }

if (hparams.n_layer == 32) { model.type = e_model::MODEL_7B; }

if (hparams.n_layer == 40) { model.type = e_model::MODEL_13B; }

if (hparams.n_layer == 60) { model.type = e_model::MODEL_30B; }

if (hparams.n_layer == 80) { model.type = e_model::MODEL_65B; }

fprintf(stderr, “%s: n_vocab = %d\n”, func, hparams.n_vocab); fprintf(stderr, “%s: n_ctx = %d\n”, func, hparams.n_ctx); fprintf(stderr, “%s: n_embd = %d\n”, func, hparams.n_embd); fprintf(stderr, “%s: n_mult = %d\n”, func, hparams.n_mult); fprintf(stderr, “%s: n_head = %d\n”, func, hparams.n_head); fprintf(stderr, “%s: n_layer = %d\n”, func, hparams.n_layer); fprintf(stderr, “%s: n_rot = %d\n”, func, hparams.n_rot); fprintf(stderr, “%s: f16 = %d\n”, func, hparams.f16); fprintf(stderr, “%s: n_ff = %d\n”, func, n_ff); fprintf(stderr, “%s: n_parts = %d\n”, func, n_parts); fprintf(stderr, “%s: type = %d\n”, func, model.type); }

// load vocab { std::string word; vocab.id_to_token.resize(model.hparams.n_vocab); std::vector tmp(64);

for (int i = 0; i < model.hparams.n_vocab; i++) { uint32_t len; fin.read((char ) &len, sizeof(len));

word.resize(len); if (len > 0) { tmp.resize(len); fin.read(tmp.data(), len); word.assign(tmp.data(), len); } else { word.clear(); }

float score; fin.read((char ) &score, sizeof(score));

vocab.token_to_id[word] = i;

auto &tok_score = vocab.id_to_token[i]; tok_score.tok = word; tok_score.score = score; } }

if (vocab_only) { return true; }

// for the big tensors, we have the option to store the data in 16-bit floats or quantized // in order to save memory and also to speed up the computation // wtype is for per-layer weights, while vtype is for other weights ggml_type wtype, vtype; switch (model.hparams.f16) { case 0: wtype = vtype = GGML_TYPE_F32; break; case 1: wtype = vtype = GGML_TYPE_F16; break; case 2: wtype = vtype = GGML_TYPE_Q4_0; break; case 3: wtype = vtype = GGML_TYPE_Q4_1; break; case 4: wtype = GGML_TYPE_Q4_1; vtype = GGML_TYPE_F16; break; default: { fprintf(stderr, “%s: invalid model file ‘%s’ (bad f16 value %d)\n”, func, fname.c_str(), model.hparams.f16); return false; } }

// map model into memory char mm_addr = NULL; model.mm_addr = mmap_file(fname.c_str(), &model.mm_length); if (model.mm_addr == NULL) { fprintf(stderr, “%s: failed to mmap ‘%s’\n”, func, fname.c_str()); return false; } mm_addr = (char )model.mm_addr; fprintf(stderr, “%s: ggml map size = %6.2f MB\n”, func, model.mm_length/(1024.01024.0));

auto & ctx = model.ctx;

size_t ctx_size = 0; { const auto &hparams = model.hparams; const int n_layer = hparams.n_layer; ctx_size += (5 + 10n_layer)256; // object overhead fprintf(stderr, “%s: ggml ctx size = %6.2f KB\n”, func, ctx_size/1024.0); }

// print memory requirements { const size_t scale = memory_type == GGML_TYPE_F32 ? 2 : 1;

// this is the total memory required to run the inference const size_t mem_required = ctx_size + model.mm_length + MEM_REQ_SCRATCH0.at(model.type) + MEM_REQ_SCRATCH1.at(model.type) + MEM_REQ_EVAL.at (model.type);

// this is the memory required by one llama_state const size_t mem_required_state = scaleMEM_REQ_KV_SELF.at(model.type);

fprintf(stderr, “%s: mem required = %7.2f MB (+ %7.2f MB per state)\n”, func, mem_required / 1024.0 / 1024.0, mem_required_state / 1024.0 / 1024.0); }

// create the ggml context { lctx.model.buf.resize(ctx_size);

struct ggml_init_params params = { /.mem_size =/ lctx.model.buf.size(), /.mem_buffer =/ lctx.model.buf.data(), /.no_alloc =/ true, };

model.ctx = ggml_init(params); if (!model.ctx) { fprintf(stderr, “%s: ggml_init() failed\n”, func); return false; } }

// prepare memory for the weights { const auto & hparams = model.hparams;

const int n_embd = hparams.n_embd; const int n_layer = hparams.n_layer; const int n_vocab = hparams.n_vocab;

model.layers.resize(n_layer);

model.tok_embeddings = ggml_new_tensor_2d(ctx, vtype, n_embd, n_vocab);

model.norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); model.output = ggml_new_tensor_2d(ctx, vtype, n_embd, n_vocab);

// map by name model.tensors[“tok_embeddings.weight”] = model.tok_embeddings;

model.tensors[“norm.weight”] = model.norm; model.tensors[“output.weight”] = model.output;

for (int i = 0; i < n_layer; ++i) { auto & layer = model.layers[i];

layer.attention_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);

layer.wq = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); layer.wk = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); layer.wv = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); layer.wo = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);

layer.ffn_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);

layer.w1 = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff); layer.w2 = ggml_new_tensor_2d(ctx, wtype, n_ff, n_embd); layer.w3 = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);

// map by name model.tensors[“layers.” + std::to_string(i) + “.attention_norm.weight”] = layer.attention_norm;

model.tensors[“layers.” + std::to_string(i) + “.attention.wq.weight”] = layer.wq; model.tensors[“layers.” + std::to_string(i) + “.attention.wk.weight”] = layer.wk; model.tensors[“layers.” + std::to_string(i) + “.attention.wv.weight”] = layer.wv; model.tensors[“layers.” + std::to_string(i) + “.attention.wo.weight”] = layer.wo;

model.tensors[“layers.” + std::to_string(i) + “.ffn_norm.weight”] = layer.ffn_norm;

model.tensors[“layers.” + std::to_string(i) + “.feed_forward.w1.weight”] = layer.w1; model.tensors[“layers.” + std::to_string(i) + “.feed_forward.w2.weight”] = layer.w2; model.tensors[“layers.” + std::to_string(i) + “.feed_forward.w3.weight”] = layer.w3; } }

std::vector<uint8_t> tmp;

if (progress_callback) { progress_callback(0.0, progress_callback_user_data); }

fprintf(stderr, “%s: loading tensors from ‘%s’\n”, func, fname.c_str());

// load weights { size_t total_size = 0; model.n_loaded = 0;

while (true) { int32_t n_dims; int32_t length; int32_t ftype;

fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims)); fin.read(reinterpret_cast<char *>(&length), sizeof(length)); fin.read(reinterpret_cast<char *>(&ftype), sizeof(ftype));

if (fin.eof()) { break; }

int32_t nelements = 1; int32_t ne[2] = { 1, 1 }; for (int i = 0; i < n_dims; ++i) { fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i])); nelements *= ne[i]; }

std::string name(length, 0); fin.read(&name[0], length);

if (model.tensors.find(name.data()) == model.tensors.end()) { fprintf(stderr, “%s: unknown tensor ‘%s’ in model file\n”, func, name.data()); return false; }

auto tensor = model.tensors[name.data()];

if (ggml_nelements(tensor) != nelements) { fprintf(stderr, “%s: tensor ‘%s’ has wrong size in model file\n”, func, name.data()); return false; } if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) { fprintf(stderr, “%s: tensor ‘%s’ has wrong shape in model file: got [%” PRId64 “, %” PRId64 “], expected [%d, %d]\n”, func, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]); return false; } if (0) { static const char * ftype_str[] = { “f32”, “f16”, “q4_0”, “q4_1”, }; fprintf(stderr, “%24s - [%5d, %5d], type = %6s\n”, name.data(), ne[0], ne[1], ftype_str[ftype]); }

switch (ftype) { case 0: // f32 case 1: // f16 break; case 2: // q4_0 case 3: // q4_1 assert(ne[0] % 64 == 0); break; default: fprintf(stderr, “%s: unknown ftype %d in model file\n”, func, ftype); return false; };

// load the tensor data into memory without copying or reading it size_t offset = fin.tellg(); size_t tensor_data_size = ggml_nbytes(tensor); offset = (offset + 31) & -32; tensor->data = mm_addr + offset; fin.seekg(offset + tensor_data_size); total_size += tensor_data_size; model.n_loaded++;

// progress if (progress_callback) { double current_progress = size_t(fin.tellg()) / double(file_size); progress_callback(current_progress, progress_callback_user_data); } }

fin.close();

fprintf(stderr, “%s: model size = %8.2f MB / num tensors = %d\n”, func, total_size/1024.0/1024.0, model.n_loaded); if (model.n_loaded == 0) { fprintf(stderr, “%s: WARN no tensors loaded from model file - assuming empty model for testing\n”, func); } else if (model.n_loaded != (int) model.tensors.size()) { fprintf(stderr, “%s: ERROR not all tensors loaded from model file - expected %zu, got %d\n”, func, model.tensors.size(), model.n_loaded); return false; } }

// loading time will be recalculate after the first eval, so // we take page faults deferred by mmap() into consideration lctx.t_load_us = ggml_time_us() - lctx.t_start_us;

if (progress_callback) { progress_callback(1.0, progress_callback_user_data); }

return true; }

here is how the model is exported :

#! /usr/bin/env python

coding=utf-8

""" Modified from: https://github.com/tloen/alpaca-lora """

import json import os

import fire import torch from peft import PeftModel from transformers import LlamaForCausalLM, LlamaTokenizer

CHECKPOINT_PARAMS = { "7b": {"dim": 4096, "multiple_of": 256, "n_heads": 32, "n_layers": 32, "norm_eps": 1e-06, "vocab_size": -1}, "13b": {"dim": 5120, "multiple_of": 256, "n_heads": 40, "n_layers": 40, "norm_eps": 1e-06, "vocab_size": -1}, "30b": {"dim": 6656, "multiple_of": 256, "n_heads": 52, "n_layers": 60, "norm_eps": 1e-06, "vocab_size": -1}, "65b": {"dim": 8192, "multiple_of": 256, "n_heads": 64, "n_layers": 80, "norm_eps": 1e-06, "vocab_size": -1}, }

def main(base_model_name_or_path: str, lora_model_name_or_path: str, output_dir: str, checkpoint_size: str = "7b"):

# Retrieve the model parameters
params = CHECKPOINT_PARAMS.get(checkpoint_size)
if params is None:
    raise ValueError(
        f"Cannot find the right model parameters for {checkpoint_size}. Please choose between {list(CHECKPOINT_PARAMS.keys())}."
    )

# tokenizer = LlamaTokenizer.from_pretrained(base_model_name_or_path)

base_model = LlamaForCausalLM.from_pretrained(
    base_model_name_or_path,
    load_in_8bit=False,
    torch_dtype=torch.float16,
    device_map={"": "cpu"},
)

lora_model = PeftModel.from_pretrained(
    base_model,
    lora_model_name_or_path,
    device_map={"": "cpu"},
    torch_dtype=torch.float16,
)

# merge weights
for layer in lora_model.base_model.model.model.layers:
    if hasattr(layer.self_attn.q_proj, "merge_weights"):
        layer.self_attn.q_proj.merge_weights = True
    if hasattr(layer.self_attn.v_proj, "merge_weights"):
        layer.self_attn.v_proj.merge_weights = True
    if hasattr(layer.self_attn.k_proj, "merge_weights"):
        layer.self_attn.k_proj.merge_weights = True
    if hasattr(layer.self_attn.o_proj, "merge_weights"):
        layer.self_attn.o_proj.merge_weights = True
    if hasattr(layer.mlp.gate_proj, "merge_weights"):
        layer.mlp.gate_proj.merge_weights = True
    if hasattr(layer.mlp.down_proj, "merge_weights"):
        layer.mlp.down_proj.merge_weights = True
    if hasattr(layer.mlp.up_proj, "merge_weights"):
        layer.mlp.up_proj.merge_weights = True

lora_model.train(False)

lora_model_sd = lora_model.state_dict()

# params = {
#     "dim": 4096,
#     "multiple_of": 256,
#     "n_heads": 32,
#     "n_layers": 32,
#     "norm_eps": 1e-06,
#     "vocab_size": -1,
# }
n_layers = params["n_layers"]
n_heads = params["n_heads"]
dim = params["dim"]
dims_per_head = dim // n_heads
base = 10000.0
inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head))

def permute(w):
    return w.view(n_heads, dim // n_heads // 2, 2, dim).transpose(1, 2).reshape(dim, dim)

def unpermute(w):
    return w.view(n_heads, 2, dim // n_heads // 2, dim).transpose(1, 2).reshape(dim, dim)

def translate_state_dict_key(k):
    k = k.replace("base_model.model.", "")
    if k == "model.embed_tokens.weight":
        return "tok_embeddings.weight"
    elif k == "model.norm.weight":
        return "norm.weight"
    elif k == "lm_head.weight":
        return "output.weight"
    elif k.startswith("model.layers."):
        layer = k.split(".")[2]
        if k.endswith(".self_attn.q_proj.weight"):
            return f"layers.{layer}.attention.wq.weight"
        elif k.endswith(".self_attn.k_proj.weight"):
            return f"layers.{layer}.attention.wk.weight"
        elif k.endswith(".self_attn.v_proj.weight"):
            return f"layers.{layer}.attention.wv.weight"
        elif k.endswith(".self_attn.o_proj.weight"):
            return f"layers.{layer}.attention.wo.weight"
        elif k.endswith(".mlp.gate_proj.weight"):
            return f"layers.{layer}.feed_forward.w1.weight"
        elif k.endswith(".mlp.down_proj.weight"):
            return f"layers.{layer}.feed_forward.w2.weight"
        elif k.endswith(".mlp.up_proj.weight"):
            return f"layers.{layer}.feed_forward.w3.weight"
        elif k.endswith(".input_layernorm.weight"):
            return f"layers.{layer}.attention_norm.weight"
        elif k.endswith(".post_attention_layernorm.weight"):
            return f"layers.{layer}.ffn_norm.weight"
        elif k.endswith("rotary_emb.inv_freq") or "lora" in k:
            return None
        else:
            print(layer, k)
            raise NotImplementedError
    else:
        print(k)
        raise NotImplementedError



new_state_dict = {}
for k, v in lora_model_sd.items():
    new_k = translate_state_dict_key(k)
    if new_k is not None:
        if "wq" in new_k or "wk" in new_k:
            new_state_dict[new_k] = unpermute(v)
        else:
            new_state_dict[new_k] = v

os.makedirs(output_dir, exist_ok=True)

Split the tensors based on layer index

n_layers_actual = len([k for k in new_state_dict.keys() if ".attention.wq.weight" in k])
part1_keys = [k for k in new_state_dict.keys() if not k.startswith("layers.") or int(k.split(".")[1]) < (n_layers_actual // 2)]
part2_keys = [k for k in new_state_dict.keys() if k not in part1_keys]

state_dict_part1 = {k: new_state_dict[k] for k in part1_keys}
state_dict_part2 = {k: new_state_dict[k] for k in part2_keys}

torch.save(state_dict_part1, output_dir + "/consolidated.00.pth")
torch.save(state_dict_part2, output_dir + "/consolidated.01.pth")
with open(output_dir + "/params.json", "w") as f:
    json.dump(params, f)

if name == "main": fire.Fire(main)

Here is the problem I have when i run the inference:

./main -m ./models/13B/ggml-model-f16.bin -n 5000 --repeat_penalty 1.0 --color -i -r “User:” -f prompts/chat-with-bob.txt -t 32 main: seed = 1681035697 llama_model_load: loading model from ‘./models/13B/ggml-model-f16.bin’ - please wait … llama_model_load: n_vocab = 32000 llama_model_load: n_ctx = 512 llama_model_load: n_embd = 5120 llama_model_load: n_mult = 256 llama_model_load: n_head = 40 llama_model_load: n_layer = 40 llama_model_load: n_rot = 128 llama_model_load: f16 = 1 llama_model_load: n_ff = 13824 llama_model_load: n_parts = 2 llama_model_load: type = 2 llama_model_load: ggml map size = 25138.72 MB llama_model_load: ggml ctx size = 101.25 KB llama_model_load: mem required = 27186.82 MB (+ 1608.00 MB per state) llama_model_load: loading tensors from ‘./models/13B/ggml-model-f16.bin’ llama_model_load: tensor ‘layers.20.attention.wq.weight’ has wrong size in model file llama_init_from_file: failed to load model main: error: failed to load model ‘./models/13B/ggml-model-f16.bin’