Skip to content
TheChatArchiveThe Chat Archive
Mistralmistral-small-3.1-24b-instruct-2503· Mistral AIHugging Face Arena Expert 5K

any fault here: void loadshardlayers(MLA_weights w, Config cfg, const char *bin_path,…

  1. User

    any fault here: void load_shard_layers(MLA_weights *w, Config *cfg, const char *bin_path, const char *meta_path) { FILE *fm = fopen(meta_path, "rb"); if (!fm) { perror(meta_path); exit(1); } fseek(fm, 0, SEEK_END); long msize = ftell(fm); fseek(fm, 0, SEEK_SET); char *mtext = malloc(msize + 1); fread(mtext, 1, msize, fm); mtext[msize] = '\0'; fclose(fm);

    cJSON *root = cJSON_Parse(mtext);
    if (!root) { fprintf(stderr, "JSON parse error\n"); exit(1); }
    
    FILE *fb = fopen(bin_path, "rb");
    if (!fb) { perror(bin_path); exit(1); }
    
    int n_tensors = cJSON_GetArraySize(root);
    for (int i = 0; i < n_tensors; i++) {
        cJSON *item      = cJSON_GetArrayItem(root, i);
        const char *name = cJSON_GetObjectItem(item, "name")->valuestring;
        cJSON *shape_arr = cJSON_GetObjectItem(item, "shape");
        if (strstr(name, ".weight_scale_inv")) continue;
    
        const char *dtype = cJSON_GetObjectItem(item, "dtype")->valuestring;
        size_t numel      = numel_from_shape(shape_arr);
        int layer         = get_layer_index(name);
    
        /* Allocate destination pointer depending on key */
        float *dst_ptr = NULL;
        int offset     = (layer >= 0) ? layer * numel : 0;
    
        if (strstr(name, "model.embed_tokens.weight")) {
            dst_ptr = w->embed_table;
        } else if (strstr(name, "input_layernorm.weight") && layer >= 0) {
            dst_ptr = w->rm_norm_weights_hs + offset;
        } else if (strstr(name, "self_attn.q_a_layernorm.weight") && layer >= 0) {
            dst_ptr = w->rm_norm_weights_q+ offset;
        } else if (strstr(name, "self_attn.q_a_proj.weight") && layer >= 0) {
            dst_ptr = w->wq_a + offset;
        } else if (strstr(name, "self_attn.q_b_proj.weight") && layer >= 0) {
            dst_ptr = w->wq_b + offset;
        } else if (strstr(name, "self_attn.kv_a_proj_with_mqa.weight") && layer >= 0) {
            dst_ptr = w->wkv_a + offset;
        } else if (strstr(name, "self_attn.kv_a_layernorm.weight") && layer >= 0) {
            dst_ptr = w->rm_norm_weights_kv;
        } else if (strstr(name, "self_attn.kv_b_proj.weight") && layer >= 0) {
            dst_ptr = w->wkv_b + offset;
        } else if (strstr(name, "self_attn.o_proj.weight") && layer >= 0) {
            dst_ptr = w->wo + offset;
        } else {
            /* Skip anything not a projection we care about */
            if (strcmp(dtype, "int8") == 0) {
                if (i + 1 < n_tensors) {
                    cJSON *next      = cJSON_GetArrayItem(root, i + 1);
                    cJSON *next_shape= cJSON_GetObjectItem(next, "shape");
                    cJSON *dim0      = cJSON_GetArrayItem(next_shape, 0);
                    cJSON *dim1      = cJSON_GetArrayItem(next_shape, 1);
                    size_t rows_size = (size_t)dim0->valuedouble;
                    size_t cols_size = (size_t)dim1->valuedouble;
                    fseek(fb,
                          numel * sizeof(int8_t) +
                          (rows_size * cols_size) * sizeof(float),
                          SEEK_CUR);
                }
            } else {
                fseek(fb, numel * sizeof(float), SEEK_CUR);
            }
            continue;
        }
    
        if (strcmp(dtype, "float32") == 0) {
            fread(dst_ptr, sizeof(float), numel, fb);
        } else if (strcmp(dtype, "int8") == 0) {
            cJSON *dim0 = cJSON_GetArrayItem(shape_arr, 0);
            cJSON *dim1 = cJSON_GetArrayItem(shape_arr, 1);
            size_t rows_size = (size_t)dim0->valuedouble;
            size_t cols_size = (size_t)dim1->valuedouble;
            int rows = (int)rows_size;
            int cols = (int)cols_size;
    
            if (!dst_ptr) {
                fprintf(stderr, "ERROR: dst_ptr is NULL for tensor %s\n", name);
                exit(1);
            }
            printf("row and col for %s: (%d, %d)\n", name, rows, cols);
            dequantize_and_load_grouped(dst_ptr, fb, cfg, rows, cols);
        } else {
            continue;
        }
    }
    
  2. Mistral Assistant

    Your load_shard_layers function looks mostly correct, but there are a few potential issues and improvements that could help ensure it works as expected. Let's go through the code step-by-step and identify any potential faults or improvements:

    1. Memory Allocation and File Reading

    char *mtext = malloc(msize + 1);
    fread(mtext, 1, msize, fm);
    mtext[msize] = '\0';
    
    • Potential Issue: If fread does not read the entire file, mtext[msize] will not be null-terminated. You should check the return value of fread to ensure it reads the expected number of bytes.
    size_t bytes_read = fread(mtext, 1, msize, fm);
    if (bytes_read != msize) {
        fprintf(stderr, "Error reading meta file\n");
        free(mtext);
        fclose(fm);
        exit(1);
    }
    mtext[msize] = '\0';
    

    2. JSON Parsing

    cJSON *root = cJSON_Parse(mtext);
    if (!root) { fprintf(stderr, "JSON parse error\n"); exit(1); }
    
    • Potential Issue: Ensure that mtext is properly null-terminated before parsing. This is already handled by setting mtext[msize] = '\0'.

    3. File Reading Loop

    for (int i = 0; i < n_tensors; i++) {
        cJSON *item = cJSON_GetArrayItem(root, i);
        const char *name = cJSON_GetObjectItem(item, "name")->valuestring;
        cJSON *shape_arr = cJSON_GetObjectItem(item, "shape");
        if (strstr(name, ".weight_scale_inv")) continue;
    
        const char *dtype = cJSON_GetObjectItem(item, "dtype")->valuestring;
        size_t numel = numel_from_shape(shape_arr);
        int layer = get_layer_index(name);
    
    • Potential Issue: Ensure that numel_from_shape and get_layer_index functions are correct and handle all edge cases.

    4. Destination Pointer Allocation

    float *dst_ptr = NULL;
    int offset = (layer >= 0) ? layer * numel : 0;
    
    • Potential Issue: Ensure that offset calculation is correct and that dst_ptr is properly initialized for each tensor.

    5. Tensor Type Handling

    if (strcmp(dtype, "float32") == 0) {
        fread(dst_ptr, sizeof(float), numel, fb);
    } else if (strcmp(dtype, "int8") == 0) {
        cJSON *dim0 = cJSON_GetArrayItem(shape_arr, 0);
        cJSON *dim1 = cJSON_GetArrayItem(shape_arr, 1);
        size_t rows_size = (size_t)dim0->valuedouble;
        size_t cols_size = (size_t)dim1->valuedouble;
        int rows = (int)rows_size;
        int cols = (int)cols_size;
    
        if (!dst_ptr) {
            fprintf(stderr, "ERROR: dst_ptr is NULL for tensor %s\n", name);
            exit(1);
        }
        printf("row and col for %s: (%d, %d)\n", name, rows, cols);
        dequantize_and_load_grouped(dst_ptr, fb, cfg, rows, cols);
    } else {
        continue;
    }
    
    • Potential Issue: Ensure that dequantize_and_load_grouped is correctly implemented and handles all edge cases. The previous discussion addressed potential issues in this function.

    6. Skipping Unnecessary Tensors

    if (strcmp(dtype, "int8") == 0) {
        if (i + 1 < n_tensors) {
            cJSON *next = cJSON_GetArrayItem(root, i + 1);
            cJSON *next_shape = cJSON_GetObjectItem(next, "shape");
            cJSON *dim0 = cJSON_GetArrayItem(next_shape, 0);
            cJSON *dim1 = cJSON_GetArrayItem(next_shape, 1);
            size_t rows_size = (size_t)dim0->valuedouble;
            size_t cols_size = (size_t)dim1->valuedouble;
            fseek(fb,
                  numel * sizeof(int8_t) +
                  (rows_size * cols_size) * sizeof(float),
                  SEEK_CUR);
        }
    } else {
        fseek(fb, numel * sizeof(float), SEEK_CUR);
    }
    continue;
    
    • Potential Issue: Ensure that the fseek calculations are correct and handle all edge cases. The fseek logic seems correct, but it's crucial to verify that the file offsets are correctly calculated.

    7. General Improvements

    • Error Handling: Add more detailed error handling and logging to help diagnose issues.
    • Bounds Checking: Add bounds checking to ensure that array indices are within valid ranges.
    • Memory Management: Ensure that all dynamically allocated memory is properly freed.

    Final Code with Improvements

    void load_shard_layers(MLA_weights *w, Config *cfg,
                           const char *bin_path,
                           const char *meta_path)
    {
        FILE *fm = fopen(meta_path, "rb");
        if (!fm) { perror(meta_path); exit(1); }
        fseek(fm, 0, SEEK_END);
        long msize = ftell(fm);
        fseek(fm, 0, SEEK_SET);
        char *mtext = malloc(msize + 1);
        size_t bytes_read = fread(mtext, 1, msize, fm);
        if (bytes_read != msize) {
            fprintf(stderr, "Error reading meta file\n");
            free(mtext);
            fclose(fm);
            exit(1);
        }
        mtext[msize] = '\0';
        fclose(fm);
    
        cJSON *root = cJSON_Parse(mtext);
        if (!root) { fprintf(stderr, "JSON parse error\n"); exit(1); }
    
        FILE *fb = fopen(bin_path, "rb");
        if (!fb) { perror(bin_path); exit(1); }
    
        int n_tensors = cJSON_GetArraySize(root);
        for (int i = 0; i < n_tensors; i++) {
            cJSON *item = cJSON_GetArrayItem(root, i);
            const char *name = cJSON_GetObjectItem(item, "name")->valuestring;
            cJSON *shape_arr = cJSON_GetObjectItem(item, "shape");
            if (strstr(name, ".weight_scale_inv")) continue;
    
            const char *dtype = cJSON_GetObjectItem(item, "dtype")->valuestring;
            size_t numel = numel_from_shape(shape_arr);
            int layer = get_layer_index(name);
    
            float *dst_ptr = NULL;
            int offset = (layer >= 0) ? layer * numel : 0;
    
            if (strstr(name, "model.embed_tokens.weight")) {
                dst_ptr = w->embed_table;
            } else if (strstr(name, "input_layernorm.weight") && layer >= 0) {
                dst_ptr = w->rm_norm_weights_hs + offset;
            } else if (strstr(name, "self_attn.q_a_layernorm.weight") && layer >= 0) {
                dst_ptr = w->rm_norm_weights_q + offset;
            } else if (strstr(name, "self_attn.q_a_proj.weight") && layer >= 0) {
                dst_ptr = w->wq_a + offset;
            } else if (strstr(name, "self_attn.q_b_proj.weight") && layer >= 0) {
                dst_ptr = w->wq_b + offset;
            } else if (strstr(name, "self_attn.kv_a_proj_with_mqa.weight") && layer >= 0) {
                dst_ptr = w->wkv_a + offset;
            } else if (strstr(name, "self_attn.kv_a_layernorm.weight") && layer >= 0) {
                dst_ptr = w->rm_norm_weights_kv;
            } else if (strstr(name, "self_attn.kv_b_proj.weight") && layer >= 0) {
                dst_ptr = w->wkv_b + offset;
            } else if (strstr(name, "self_attn.o_proj.weight") && layer >= 0) {
                dst_ptr = w->wo + offset;
            } else {
                if (strcmp(dtype, "int8") == 0) {
                    if (i + 1 < n_tensors) {
                        cJSON *next = cJSON_GetArrayItem(root, i + 1);
                        cJSON *next_shape = cJSON_GetObjectItem(next, "shape");
                        cJSON *dim0 = cJSON_GetArrayItem(next_shape, 0);
                        cJSON *dim1 = cJSON_GetArrayItem(next_shape, 1);
                        size_t rows_size = (size_t)dim0->valuedouble;
                        size_t cols_size = (size_t)dim1->valuedouble;
                        fseek(fb,
                              numel * sizeof(int8_t) +
                              (rows_size * cols_size) * sizeof(float),
                              SEEK_CUR);
                    }
                } else {
                    fseek(fb, numel * sizeof(float), SEEK_CUR);
                }
                continue;
            }
    
            if (strcmp(dtype, "float32") == 0) {
                fread(dst_ptr, sizeof(float), numel, fb);
            } else if (strcmp(dtype, "int8") == 0) {
                cJSON *dim0 = cJSON_GetArrayItem(shape_arr, 0);
                cJSON *dim1 = cJSON_GetArrayItem(shape_arr, 1);
                size_t rows_size = (size_t)dim0->valuedouble;
                size_t cols_size = (size_t)dim1->valuedouble;
                int rows = (int)rows_size;
                int cols = (int)cols_size;
    
                if (!dst_ptr) {
                    fprintf(stderr, "ERROR: dst_ptr is NULL for tensor %s\n", name);
                    exit(1);
                }
                printf("row and col for %s: (%d, %d)\n", name, rows, cols);
                dequantize_and_load_grouped(dst_ptr, fb, cfg, rows, cols);
            } else {
                continue;
            }
        }
    
        free(mtext);
        fclose(fb);
    }
    

    Conclusion

    The load_shard_layers function looks mostly correct, but there are a few potential issues and improvements that can be made. The main potential fault lies in the dequantize_and_load_grouped function, which has been addressed in the previous discussion. Ensure that all edge cases are handled and that the file offsets are correctly calculated.

  3. User

    how does cJSON does this: what kind of a C syntax is this cJSON_ArrayForEach, is it a lambda? does C support lambda?

  4. Mistral Assistant

    cJSON is a popular C library for parsing and generating JSON data. It provides a variety of functions to work with JSON objects, arrays, and values. The cJSON_ArrayForEach function is a macro that iterates over each element in a JSON array, applying a given function to each element.

    cJSON_ArrayForEach

    The cJSON_ArrayForEach macro is defined in the cJSON.h header file. It is not a lambda function; it is a macro that uses a loop to iterate over the elements of a JSON array. Here's a simplified version of how it might be defined:

    #define cJSON_ArrayForEach(array, callback) \
        for (cJSON *item = (array)->child; item; item = item->next) \
            callback(item)
    

    This macro takes two arguments:

    1. array: A pointer to a cJSON object that represents a JSON array.
    2. callback: A function that takes a single cJSON pointer as an argument and performs some operation on it.

    Example Usage

    Here's an example of how you might use cJSON_ArrayForEach to iterate over the elements of a JSON array and print each element:

    #include <stdio.h>
    #include <cJSON.h>
    
    // Callback function to print each element
    void print_element(cJSON *item) {
        printf("%s\n", item->valuestring);
    }
    
    int main() {
        const char *json_string = "[\"element1\", \"element2\", \"element3\"]";
        cJSON *root = cJSON_Parse(json_string);
        if (root == NULL) {
            fprintf(stderr, "Error parsing JSON\n");
            return 1;
        }
    
        // Use cJSON_ArrayForEach to iterate over the array
        cJSON_ArrayForEach(root, print_element);
    
        // Clean up
        cJSON_Delete(root);
        return 0;
    }
    

    In this example, print_element is a callback function that takes a cJSON pointer as an argument and prints the string value of the JSON element. The cJSON_ArrayForEach macro iterates over each element in the JSON array and calls print_element for each element.

    C and Lambda Functions

    C does not natively support lambda functions like C++ does. However, you can achieve similar functionality using function pointers or macros. The cJSON_ArrayForEach macro is an example of how you can use a macro to iterate over an array and apply a function to each element.

    If you need more complex behavior, you can define a function and pass it as a callback, as shown in the example above. For simple cases, macros can be a convenient way to achieve similar results.

    Summary

    • cJSON_ArrayForEach is a macro that iterates over the elements of a JSON array and applies a given function to each element.
    • It is not a lambda function; it uses a loop to iterate over the elements.
    • C does not natively support lambda functions, but you can use function pointers or macros to achieve similar behavior.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.