Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0314· OpenAIAi2 WildChat-4.8M (AI2)

void extractint4weighttofloat(void weight, void scale_list, void out, int n, int m) {…

  1. User

    void extract_int4_weight_to_float(void weight, void scale_list, void out, int n, int m) { for(int i=0;i<n;i++) { for(int j=0;j<m;j++) { ((float)(out)) = ((float*)(scale_list)) * (((char)(weight)) >> 4); out += sizeof(float); ((float)(out)) = ((float)(scale_list)) * (((char)(((unsigned char)(weight)) << 4))>> 4); out += sizeof(float); weight += sizeof(char); } scale_list += sizeof(float); } } windows CPU环境下,这个代码要怎么改,才能提高推理速度,不改变原来函数的功能,写出改造后的代码

  2. Assistant

    在Windows CPU环境下,为了提高推理速度,我们可以采用多线程并发计算。以下是改进后的代码示例,使用C++ 11的std::thread库实现并发计算。

    #include <thread>
    
    static const int NUM_THREADS = std::thread::hardware_concurrency();  // 获取硬件支持的最大并发数
    
    struct TaskData {
        void *weight;
        void *scale_list;
        void *out;
        int start;
        int end;
        int m;
    };
    
    void extract_int4_weight_to_float_thread(TaskData *data) {
        for (int i = data->start; i < data->end; ++i) {
            for (int j = 0; j < data->m; ++j) {
                (*(float *)(data->out)) = (*(float *)(data->scale_list)) * ((*(char *)(data->weight)) >> 4);
                data->out += sizeof(float);
                (*(float *)(data->out)) = (*(float *)(data->scale_list)) * (((char)((*(unsigned char *)(data->weight)) << 4)) >> 4);
                data->out += sizeof(float);
                data->weight += sizeof(char);
            }
            data->scale_list += sizeof(float);
        }
    }
    
    void extract_int4_weight_to_float(void *weight, void *scale_list, void *out, int n, int m) {
        std::vector<std::thread> threads(NUM_THREADS);
        std::vector<TaskData> task_data(NUM_THREADS);
    
        int chunk_size = n / NUM_THREADS;
    
        for (int i = 0; i < NUM_THREADS; ++i) {
            task_data[i] = {weight, scale_list, out, i * chunk_size,
                            (i == NUM_THREADS - 1) ? n : (i + 1) * chunk_size, m};
            weight += chunk_size * m * sizeof(char);
            scale_list += chunk_size * sizeof(float);
            out += chunk_size * m * sizeof(float);
            threads[i] = std::thread(extract_int4_weight_to_float_thread, &task_data[i]);
        }
    
        for (auto &t : threads) {
            t.join();
        }
    }
    
    

    通过使用多线程并发计算,你可以在多核CPU上实现更高的计算速度。请注意,这段代码使用了C++ 11特性,你需要使用支持C++ 11的编译器进行编译。

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.