Skip to content
TheChatArchiveThe Chat Archive
o1-preview-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

#pragma once #include <cudaruntime.h> #include <cublasv2.h> #include <stdio.h> #include…

  1. User

    #pragma once

    #include <cuda_runtime.h> #include <cublas_v2.h> #include <stdio.h> #include <stdlib.h>

    #ifndef FETCH_FLOAT4 #define FETCH_FLOAT4(pointer) (reinterpret_cast<float4 *>(&(pointer))[0]) #endif #ifndef DIV_UP #define DIV_UP(m, n) ((m + n - 1) / n) #endif #define GROUP_SIZE 8 #define WARP_SIZE 32

    namespace kernel9 {

    template<int BM, int BN, int BK>
    __device__ __forceinline__ void gmem_to_smem(float* A, float* B, float smem_a[][BK][BM], float smem_b[][BK][BN], float ldreg_a[][4], float ldreg_b[][4], int a_smem_rounds, int a_stride, int a_smem_x, int a_smem_y, int b_smem_rounds, int b_stride, int b_smem_y, int b_smem_x, int phase)
    {
    

    #pragma unroll // A: global -> reg buffer for (int i = 0; i < a_smem_rounds; ++i) { FETCH_FLOAT4(ldreg_a[i]) = FETCH_FLOAT4(A[i * a_stride]); smem_a[phase][a_smem_x][a_smem_y + i * a_stride] = ldreg_a[i][0]; smem_a[phase][a_smem_x + 1][a_smem_y + i * a_stride] = ldreg_a[i][1]; smem_a[phase][a_smem_x + 2][a_smem_y + i * a_stride] = ldreg_a[i][2]; smem_a[phase][a_smem_x + 3][a_smem_y + i * a_stride] = ldreg_a[i][3]; } #pragma unroll // B: global -> reg buffer for (int i = 0; i < b_smem_rounds; ++i) { FETCH_FLOAT4(ldreg_b[i]) = FETCH_FLOAT4(B[i * b_stride]); FETCH_FLOAT4(smem_b[phase][b_smem_y][b_smem_x + i * b_stride]) = FETCH_FLOAT4(ldreg_b[i]); } }

    __device__ __forceinline__ void gmem_to_reg(float* A, float* B, float ldreg_a[][4], float ldreg_b[][4], int a_smem_rounds, int a_stride, int b_smem_rounds, int b_stride)
    {
    

    #pragma unroll // A: global -> reg buffer for (int i = 0; i < a_smem_rounds; ++i) { FETCH_FLOAT4(ldreg_a[i]) = FETCH_FLOAT4(A[i * a_stride]); } #pragma unroll // B: global -> reg buffer for (int i = 0; i < b_smem_rounds; ++i) { FETCH_FLOAT4(ldreg_b[i]) = FETCH_FLOAT4(B[i * b_stride]); } }

    template<int BM, int BN, int BK>
    __device__ __forceinline__ void reg_to_smem(float smem_a[][BK][BM], float smem_b[][BK][BN], float ldreg_a[][4], float ldreg_b[][4], int a_smem_rounds, int a_stride, int a_smem_x, int a_smem_y, int b_smem_rounds, int b_stride, int b_smem_y, int b_smem_x, int phase)
    {
    

    #pragma unroll // A: reg buffer -> smem for (int i = 0; i < a_smem_rounds; ++i) { // note that this is uncoalesce memory write, and only 4 floats * 4 byte/float = 16 bytes per write smem_a[phase][a_smem_x][a_smem_y + i * a_stride] = ldreg_a[i][0]; smem_a[phase][a_smem_x + 1][a_smem_y + i * a_stride] = ldreg_a[i][1]; smem_a[phase][a_smem_x + 2][a_smem_y + i * a_stride] = ldreg_a[i][2]; smem_a[phase][a_smem_x + 3][a_smem_y + i * a_stride] = ldreg_a[i][3]; } #pragma unroll // B: reg buffer -> smem for (int i = 0; i < b_smem_rounds; ++i) { FETCH_FLOAT4(smem_b[phase][b_smem_y][b_smem_x + i * b_stride]) = FETCH_FLOAT4(ldreg_b[i]); } }

    template<int BM, int BN, int BK, int TM, int TN>
    __device__ __forceinline__ void smem_to_frag(float frag_a[][TM], float frag_b[][TN], float smem_a[][BK][BM], float smem_b[][BK][BN], int frag_phase, int smem_phase, int bk)
    {
        // int swizzel_id = (threadIdx.x / 4) % 2;
    

    #pragma unroll for (int i = 0; i < TM; i += 4) { // int swizzel_i = ((i / 4) ^ swizzel_id) * 4; FETCH_FLOAT4(frag_a[frag_phase][i]) = FETCH_FLOAT4(smem_a[smem_phase][bk][threadIdx.y * TM + i]); } #pragma unroll for (int i = 0; i < TN; i += 4) { // int swizzel_i = ((i / 4) ^ swizzel_id) * 4; FETCH_FLOAT4(frag_b[frag_phase][i]) = FETCH_FLOAT4(smem_b[smem_phase][bk][threadIdx.x * TN + i]); } }

    } // namespace kernel 9

    // This function assumes B is already transposed template <const int BM, const int BN, const int BK, const int TM, const int TN, const int THREAD_NUMS> global void launch_bounds(THREAD_NUMS, 2) mysgemm_v9(int M, int N, int K, float alpha, float* A, float* B, float beta, float* C) { printf("Here\n"); int bx = blockIdx.x; int by = blockIdx.y; constexpr int threads_per_block = BM / TM * BN / TN; constexpr int a_ele_per_thread_smem = BM * BK / threads_per_block; constexpr int b_ele_per_thread_smem = BK * BN / threads_per_block; constexpr int a_smem_rounds = a_ele_per_thread_smem / 4; constexpr int b_smem_rounds = b_ele_per_thread_smem / 4; constexpr int a_threads_per_row_per_round = BK / 4; int a_stride = threads_per_block / a_threads_per_row_per_round * K; constexpr int b_threads_per_row_per_round = BN / 4; int b_stride = threads_per_block / b_threads_per_row_per_round * N; // int tid = threadIdx.y * blockDim.x + threadIdx.x; // int lane_id = tid % 32; int a_smem_x = ((threadIdx.y * blockDim.x + threadIdx.x) % a_threads_per_row_per_round) * 4; int a_smem_y = (threadIdx.y * blockDim.x + threadIdx.x) / a_threads_per_row_per_round; int b_smem_x = ((threadIdx.y * blockDim.x + threadIdx.x) % b_threads_per_row_per_round) * 4; int b_smem_y = (threadIdx.y * blockDim.x + threadIdx.x) / b_threads_per_row_per_round;

    static_assert((BM * BK) % threads_per_block == 0, "assert");
    static_assert((BK * BN) % threads_per_block == 0, "assert");
    static_assert(a_ele_per_thread_smem % 4 == 0, "assert");
    static_assert(b_ele_per_thread_smem % 4 == 0, "assert");
    static_assert(BK % 4 == 0, "assert");
    static_assert(BN % 4 == 0, "assert");
    static_assert(threads_per_block / a_threads_per_row_per_round >= 1, "assert"); // at least cover a row per round
    static_assert(threads_per_block / b_threads_per_row_per_round >= 1, "assert"); // at least cover a row per round
    static_assert(TN % 4 == 0, "assert"); // at least 4 elements per thread and TN is a multiple of 4
    
    float accum[TM][TN] = { 0. };
    
    __shared__ float smem_a[2][BK][BM]; // a transposed version of A block
    __shared__ float smem_b[2][BK][BN];
    
    // register for loading from global mem to smem
    float ldreg_a[a_smem_rounds][4];
    float ldreg_b[b_smem_rounds][4];
    
    // fragment/register for computation
    float frag_a[2][TM];
    float frag_b[2][TN];
    
    // move A to thread start
    A = &A[by * BM * K + a_smem_y * K + a_smem_x];
    B = &B[b_smem_y * N + bx * BN + b_smem_x];
    
    // 1.1 fetch from global to smem, use register as buffer
    kernel9::gmem_to_smem<BM, BN, BK>(A, B, smem_a, smem_b, ldreg_a, ldreg_b, a_smem_rounds, a_stride, a_smem_x, a_smem_y, b_smem_rounds, b_stride, b_smem_y, b_smem_x, 0);
    __syncthreads(); // need the sync such that the following fragment can be obtained
    
    // 1.2 load 0 round of smem->frag
    kernel9::smem_to_frag<BM, BN, BK, TM, TN>(frag_a, frag_b, smem_a, smem_b, 0, 0, 0); // load first batch of frag from first block of smem
    int smem_write_index = 1; // next index of smems to write to
    int smem_read_index; // read is current write
    
    // 2. start the blockwise loop
    for (int k = 0; k < K / BK; ++k)
    {
        // 2.0 fetch from global to smem, use register as buffer
        if (k + 1 < K / BK) {
            A += BK; // every iteration, A moves BK to the right
            B += N * BK; // every iteration, B moves BK * N down
            kernel9::gmem_to_reg(A, B, ldreg_a, ldreg_b, a_smem_rounds, a_stride, b_smem_rounds, b_stride); // only load to reg, this is non-blocking
        }
        // 2.1 use the frag already loaded to compute the outer product, note that we do register prefetching here
    
        smem_read_index = smem_write_index ^ 1;
    

    #pragma unroll for (int b_k = 1; b_k < BK; ++b_k) // load one sub row at a time from smem to frag { kernel9::smem_to_frag<BM, BN, BK, TM, TN>(frag_a, frag_b, smem_a, smem_b, b_k % 2, smem_read_index, b_k); #pragma unroll for (int i = 0; i < TM; ++i) { // outer product for the previous prefetched frag #pragma unroll for (int j = 0; j < TN; ++j) { accum[i][j] += frag_a[(b_k - 1) % 2][i] * frag_b[(b_k - 1) % 2][j]; } } } // 2.2 if there's next block, start loading from reg to smem if (k + 1 < K / BK) { kernel9::reg_to_smem<BM, BN, BK>(smem_a, smem_b, ldreg_a, ldreg_b, a_smem_rounds, a_stride, a_smem_x, a_smem_y, b_smem_rounds, b_stride, b_smem_y, b_smem_x, smem_write_index); __syncthreads(); // prefetch a round of fragments from the current write, this will be blocking kernel9::smem_to_frag<BM, BN, BK, TM, TN>(frag_a, frag_b, smem_a, smem_b, 0, smem_write_index, 0); smem_write_index ^= 1; // update next write } #pragma unroll for (int i = 0; i < TM; ++i) { // one last round of outer product because we have only done BK - 1 products #pragma unroll for (int j = 0; j < TN; ++j) { accum[i][j] += frag_a[(BK - 1) % 2][i] * frag_b[(BK - 1) % 2][j]; } } }

    // 3. put the accumulate value down to C
    // move C to thread tile start
    C = &C[(by * BM + threadIdx.y * TM) * N + bx * BN + threadIdx.x * TN];
    

    #pragma unroll for (int i = 0; i < TM; ++i) { #pragma unroll for (int j = 0; j < TM; j += 4) { float4 tmp = FETCH_FLOAT4(C[i * N + j]); tmp.x = alpha * accum[i][j] + beta * tmp.x; tmp.y = alpha * accum[i][j + 1] + beta * tmp.y; tmp.z = alpha * accum[i][j + 2] + beta * tmp.z; tmp.w = alpha * accum[i][j + 3] + beta * tmp.w; FETCH_FLOAT4(C[i * N + j]) = tmp; } } } #include #include #include #include #include <cuda_runtime.h> #include "pipelined_gemm.cuh"

    #include <cuda_runtime.h> #include <cublas_v2.h> #include <stdio.h> #include <stdlib.h> #include <math.h>

    // Include your kernel code here or in a header file // #include "mysgemm_v9.cuh"

    #ifndef FETCH_FLOAT4 #define FETCH_FLOAT4(pointer) (reinterpret_cast<float4 *>(&(pointer))[0]) #endif

    #ifndef DIV_UP #define DIV_UP(m, n) ((m + n - 1) / n) #endif

    #define GROUP_SIZE 8 #define WARP_SIZE 32

    // Define the kernel9 namespace and the mysgemm_v9 kernel here // ... [Include the kernel code provided above]

    int main() { // Define matrix dimensions and constants const int M = 64; const int N = 64; const int K = 64; const int BM = 64; const int BN = 64; const int BK = 16; const int TM = 8; const int TN = 8; const int THREAD_NUMS = (BM / TM) * (BN / TN); // Should equal 64

    // Allocate host memory for matrices A, B, C, and reference C
    float* h_A = (float*)malloc(M * K * sizeof(float));
    float* h_B = (float*)malloc(K * N * sizeof(float));
    float* h_B_transposed = (float*)malloc(N * K * sizeof(float)); // B transposed
    float* h_C = (float*)malloc(M * N * sizeof(float));
    float* h_C_ref = (float*)malloc(M * N * sizeof(float)); // Reference result
    
    // Initialize matrices A and B with random values
    for (int i = 0; i < M * K; i++) {
        h_A[i] = static_cast<float>(rand()) / RAND_MAX;
    }
    for (int i = 0; i < K * N; i++) {
        h_B[i] = static_cast<float>(rand()) / RAND_MAX;
    }
    
    // Transpose B to get h_B_transposed
    for (int k = 0; k < K; k++) {
        for (int n = 0; n < N; n++) {
            h_B_transposed[n * K + k] = h_B[k * N + n];
        }
    }
    
    // Initialize C and reference C to zero
    for (int i = 0; i < M * N; i++) {
        h_C[i] = 0.0f;
        h_C_ref[i] = 0.0f;
    }
    
    // Compute reference result h_C_ref = alpha * A * B + beta * C on CPU
    float alpha = 1.0f;
    float beta = 0.0f;
    for (int m = 0; m < M; m++) {
        for (int n = 0; n < N; n++) {
            float sum = 0.0f;
            for (int k = 0; k < K; k++) {
                sum += h_A[m * K + k] * h_B[k * N + n];
            }
            h_C_ref[m * N + n] = alpha * sum + beta * h_C_ref[m * N + n];
        }
    }
    
    // Allocate device memory and copy data from host to device
    float* d_A, * d_B, * d_C;
    cudaMalloc((void**)&d_A, M * K * sizeof(float));
    cudaMalloc((void**)&d_B, N * K * sizeof(float)); // Note: N * K due to transposed B
    cudaMalloc((void**)&d_C, M * N * sizeof(float));
    
    cudaMemcpy(d_A, h_A, M * K * sizeof(float), cudaMemcpyHostToDevice);
    cudaMemcpy(d_B, h_B_transposed, N * K * sizeof(float), cudaMemcpyHostToDevice);
    cudaMemcpy(d_C, h_C, M * N * sizeof(float), cudaMemcpyHostToDevice);
    
    // Set up grid and block dimensions
    dim3 block(BM * BN / (TM * TN));
    dim3 blockDim(8, 8); // blockDim.x * blockDim.y should equal THREAD_NUMS
    dim3 gridDim(DIV_UP(N, BN), DIV_UP(M, BM)); // grid dimensions
    
    // Launch the kernel with appropriate template parameters
    mysgemm_v9<BM, BN, BK, TM, TN, THREAD_NUMS> << <gridDim, blockDim>> > (M, N, K, alpha, d_A, d_B, beta, d_C);
    
    // Copy the result back to host
    cudaMemcpy(h_C, d_C, M * N * sizeof(float), cudaMemcpyDeviceToHost);
    
    // Verify the result by comparing h_C and h_C_ref
    float max_error = 0.0f;
    float sum_error = 0.0f;
    for (int i = 0; i < M * N; i++) {
        float error = fabs(h_C_ref[i] - h_C[i]);
        if (error > max_error) {
            max_error = error;
        }
        sum_error += error;
    }
    printf("Max error: %f\n", max_error);
    printf("Sum of errors: %f\n", sum_error);
    
    // Clean up and free allocated memory
    free(h_A);
    free(h_B);
    free(h_B_transposed);
    free(h_C);
    free(h_C_ref);
    cudaFree(d_A);
    cudaFree(d_B);
    cudaFree(d_C);
    
    return 0;
    

    } my kernel is not being launched at all

  2. Assistant

    The issue you're encountering—where your kernel doesn't seem to be launching—is likely due to an error occurring during the kernel launch or execution. Since CUDA errors are not automatically reported unless explicitly checked, they can silently cause the kernel to fail without any visible indication.

    Here's how you can diagnose and fix the problem:

    1. Add Error Checking After CUDA API Calls:

    After each CUDA API call, including memory allocations, memory copies, and kernel launches, you should check for errors. CUDA provides the cudaGetLastError() function to retrieve the last error from a runtime call. Here's how you can modify your code:

    // After memory allocation
    cudaMalloc((void**)&d_A, M * K * sizeof(float));
    cudaError_t err = cudaGetLastError();
    if (err != cudaSuccess) {
        fprintf(stderr, "CUDA error allocating memory for d_A: %s\n", cudaGetErrorString(err));
        return -1;
    }
    
    // Similarly check after cudaMalloc for d_B and d_C
    
    // After memory copy to device
    cudaMemcpy(d_A, h_A, M * K * sizeof(float), cudaMemcpyHostToDevice);
    err = cudaGetLastError();
    if (err != cudaSuccess) {
        fprintf(stderr, "CUDA error copying h_A to d_A: %s\n", cudaGetErrorString(err));
        return -1;
    }
    
    // Similarly check after cudaMemcpy for d_B and d_C
    
    // Before kernel launch
    cudaDeviceSynchronize(); // Ensure previous operations are completed
    
    // Launch the kernel
    mysgemm_v9<BM, BN, BK, TM, TN, THREAD_NUMS><<<gridDim, blockDim>>>(M, N, K, alpha, d_A, d_B, beta, d_C);
    
    // After kernel launch
    cudaDeviceSynchronize(); // Wait for the kernel to finish
    err = cudaGetLastError();
    if (err != cudaSuccess) {
        fprintf(stderr, "CUDA error launching kernel: %s\n", cudaGetErrorString(err));
        return -1;
    }
    

    2. Correct the Kernel Launch Parameters:

    Ensure that you're passing the correct block and grid dimensions. In your code, it seems there's a confusion between block and blockDim. Here's the corrected version:

    // Set up block dimensions
    dim3 blockDim(8, 8); // blockDim.x * blockDim.y should equal THREAD_NUMS (64)
    
    // Set up grid dimensions
    dim3 gridDim(DIV_UP(N, BN), DIV_UP(M, BM)); // For N=64, BN=64, M=64, BM=64, this will be (1,1)
    

    Make sure you're passing gridDim and blockDim correctly:

    mysgemm_v9<BM, BN, BK, TM, TN, THREAD_NUMS><<<gridDim, blockDim>>>(M, N, K, alpha, d_A, d_B, beta, d_C);
    

    3. Verify the Kernel Execution:

    The printf("Here\n"); statement inside your kernel should print output if the kernel is executing. If you don't see this output, it's an indication that the kernel isn't running. Reasons could include:

    • Kernel launch parameters are incorrect.
    • There's an error during the kernel execution.
    • The device doesn't support printf inside kernels (requires compute capability 2.0 or higher).

    4. Check for Synchronization Issues:

    Ensure that the CPU waits for the GPU to finish executing the kernel before proceeding. This can be done using cudaDeviceSynchronize():

    // After kernel launch
    cudaDeviceSynchronize(); // Wait for the kernel to finish
    err = cudaGetLastError();
    if (err != cudaSuccess) {
        fprintf(stderr, "CUDA error during kernel execution: %s\n", cudaGetErrorString(err));
        return -1;
    }
    

    5. Use CUDA-MEMCHECK Tool:

    Run your program using the CUDA-MEMCHECK tool to detect invalid memory accesses and other errors:

    cuda-memcheck ./your_program
    

    6. Ensure Proper Compilation:

    Make sure your code is being compiled with the correct compute capability and necessary compiler flags. For example:

    nvcc -arch=sm_60 -o your_program your_program.cu
    

    Replace sm_60 with the compute capability of your GPU.

    7. Check for Return Values of CUDA Calls:

    Besides cudaMalloc and cudaMemcpy, ensure you check the return values of all CUDA runtime calls, which can silently fail if not handled.

    8. Increase the CUDA Printf Buffer Size (If Necessary):

    If you're using printf inside the kernel and not seeing output, you might need to increase the device's print buffer size:

    cudaDeviceSetLimit(cudaLimitPrintfFifoSize, 10 * 1024 * 1024);
    

    Add this before launching the kernel.

    9. Review the Kernel Code for Errors:

    Sometimes, errors within the kernel code, such as out-of-bounds memory accesses, can cause the kernel to fail silently. Carefully review your kernel code for potential issues.

    10. Sample Code with Error Checking:

    Here's how you can modify your main function to include error checking:

    int main() {
        // ... [Your existing code]
    
        // Allocate device memory and copy data from host to device
        float* d_A, * d_B, * d_C;
        cudaError_t err = cudaMalloc((void**)&d_A, M * K * sizeof(float));
        if (err != cudaSuccess) {
            fprintf(stderr, "CUDA error allocating memory for d_A: %s\n", cudaGetErrorString(err));
            return -1;
        }
        // Similarly for d_B and d_C
    
        // Copy data to device
        err = cudaMemcpy(d_A, h_A, M * K * sizeof(float), cudaMemcpyHostToDevice);
        if (err != cudaSuccess) {
            fprintf(stderr, "CUDA error copying h_A to d_A: %s\n", cudaGetErrorString(err));
            return -1;
        }
        // Similarly for h_B_transposed to d_B and h_C to d_C
    
        // Kernel launch parameters
        dim3 blockDim(8, 8);
        dim3 gridDim(DIV_UP(N, BN), DIV_UP(M, BM));
    
        // Launch the kernel
        mysgemm_v9<BM, BN, BK, TM, TN, THREAD_NUMS><<<gridDim, blockDim>>>(M, N, K, alpha, d_A, d_B, beta, d_C);
    
        // Wait for kernel to finish and check for errors
        cudaDeviceSynchronize();
        err = cudaGetLastError();
        if (err != cudaSuccess) {
            fprintf(stderr, "CUDA error during kernel execution: %s\n", cudaGetErrorString(err));
            return -1;
        }
    
        // Copy the result back to host
        err = cudaMemcpy(h_C, d_C, M * N * sizeof(float), cudaMemcpyDeviceToHost);
        if (err != cudaSuccess) {
            fprintf(stderr, "CUDA error copying d_C to h_C: %s\n", cudaGetErrorString(err));
            return -1;
        }
    
        // ... [Rest of your code]
    
        return 0;
    }
    

    11. Re-Compile and Run:

    After making these changes, recompile your code and run it. If there's an error, it should now be reported, allowing you to fix the underlying issue.

    Summary:

    By adding error checking, synchronizing after kernel launches, and verifying that the kernel launch parameters are correct, you should be able to diagnose why your kernel isn't executing. The most common cause is an unnoticed error during the kernel launch or execution, which can be revealed by the methods described above.