Skip to content
TheChatArchiveThe Chat Archive
Claudeclaude-3-5-sonnet-20241022· AnthropicHugging Face LMArena Human Preference 140K

Super and now implement a format where 16 values are stored as 4 bit, with a shared 6…

From a Chatbot Arena battleThe same person sent this prompt to two anonymous models at once. Compare it with Geminigemini-2.5-pro-preview-03-25’s answer.
  1. User

    Super and now implement a format where 16 values are stored as 4 bit, with a shared 6 bits scale (on a log scale from 1..1024) and a 2 bit offset(-2,-1,+1,+2).

  2. Claude Assistant

    I'll create a scaled_int4_codec that implements this format. The 16 values will be stored in 10 bytes total:

    • 1 byte for scale (6 bits) and offset (2 bits)
    • 8 bytes for the 16 4-bit values
    • 1 byte unused (for alignment)

    Here's the implementation:

    struct scaled_int4_codec {
        // Helper functions for scale/offset encoding/decoding
        static constexpr float decode_scale(uint8_t scale_bits) {
            // 6 bits give us 64 possible values
            // We'll use a log scale from 2^0 to 2^10 (1 to 1024)
            return std::pow(2.0f, (scale_bits * 10.0f) / 63.0f);
        }
    
        static constexpr uint8_t encode_scale(float scale) {
            // Clamp scale to [1, 1024]
            scale = std::clamp(scale, 1.0f, 1024.0f);
            // Convert to 6-bit value
            float log2_scale = std::log2(scale);
            return static_cast<uint8_t>((log2_scale * 63.0f) / 10.0f);
        }
    
        static constexpr float decode_offset(uint8_t offset_bits) {
            // 2 bits encode: -2, -1, +1, +2
            static constexpr float offsets[] = {-2.0f, -1.0f, 1.0f, 2.0f};
            return offsets[offset_bits];
        }
    
        static constexpr uint8_t encode_offset(float offset) {
            // Find closest match to -2, -1, 1, 2
            if (offset <= -1.5f) return 0;      // -2
            else if (offset <= 0.0f) return 1;  // -1
            else if (offset <= 1.5f) return 2;  // +1
            else return 3;                      // +2
        }
    
        // Structure representing the packed format
        struct packed_data {
            uint8_t scale_offset;  // 6 bits scale, 2 bits offset
            uint8_t values[8];    // 16 4-bit values
            uint8_t padding;      // Alignment padding
        };
        static_assert(sizeof(packed_data) == 10, "packed_data must be 10 bytes");
    
        template <typename TargetValueType>
        static constexpr bundle<TargetValueType, 16> load(const void* src)
            requires std::is_same_v<TargetValueType, float>
        {
            const packed_data* data = static_cast<const packed_data*>(src);
            
            // Extract scale and offset
            uint8_t scale_bits = data->scale_offset >> 2;  // Upper 6 bits
            uint8_t offset_bits = data->scale_offset & 0x3;  // Lower 2 bits
            
            float scale = decode_scale(scale_bits);
            float offset = decode_offset(offset_bits);
    
            // Create result bundle
            bundle<float, 16> result;
            alignas(16) float unpacked[16];
    
            // Unpack each byte into two float values
            for (size_t i = 0; i < 8; ++i) {
                uint8_t byte = data->values[i];
                
                // Extract low and high nibbles
                int8_t lo_val = byte & 0x0F;
                int8_t hi_val = (byte >> 4) & 0x0F;
                
                // Convert to -8..7 range
                lo_val = (lo_val & 0x08) ? (lo_val | 0xF0) : lo_val;
                hi_val = (hi_val & 0x08) ? (hi_val | 0xF0) : hi_val;
                
                // Apply scale and offset
                unpacked[i * 2] = static_cast<float>(lo_val) * scale + offset;
                unpacked[i * 2 + 1] = static_cast<float>(hi_val) * scale + offset;
            }
    
            // Load unpacked values into Neon vector
            result.data = vld1q_f32_x4(unpacked);
            return result;
        }
    
        template <typename SourceValueType>
        static constexpr void store(const bundle<SourceValueType, 16>& bundle_data, void* dst)
            requires std::is_same_v<SourceValueType, float>
        {
            packed_data* out = static_cast<packed_data*>(dst);
            alignas(16) float values[16];
            
            // Store vector data to array for processing
            vst1q_f32_x4(values, bundle_data.data);
    
            // First pass: find scale and offset
            float min_val = values[0];
            float max_val = values[0];
            for (int i = 1; i < 16; ++i) {
                min_val = std::min(min_val, values[i]);
                max_val = std::max(max_val, values[i]);
            }
    
            // Calculate optimal scale and offset
            float data_range = max_val - min_val;
            float center = (max_val + min_val) / 2.0f;
            
            // We need to map to -8..7 range
            float scale = data_range / 15.0f;  // 15 is the range of our int4 (-8..7)
            
            // Find best offset from {-2, -1, 1, 2}
            float best_offset = 1.0f;  // Default
            float min_error = std::numeric_limits<float>::max();
            
            for (float test_offset : {-2.0f, -1.0f, 1.0f, 2.0f}) {
                float error = std::abs(center - test_offset);
                if (error < min_error) {
                    min_error = error;
                    best_offset = test_offset;
                }
            }
    
            // Store scale and offset
            uint8_t scale_bits = encode_scale(scale);
            uint8_t offset_bits = encode_offset(best_offset);
            out->scale_offset = (scale_bits << 2) | offset_bits;
    
            // Second pass: quantize values
            scale = decode_scale(scale_bits);  // Use actual encoded scale
            float offset = decode_offset(offset_bits);  // Use actual encoded offset
    
            for (size_t i = 0; i < 8; ++i) {
                // Process two values at a time
                float val1 = values[i * 2];
                float val2 = values[i * 2 + 1];
    
                // Remove offset and scale
                int8_t quant1 = static_cast<int8_t>(std::round((val1 - offset) / scale));
                int8_t quant2 = static_cast<int8_t>(std::round((val2 - offset) / scale));
    
                // Clamp to int4 range (-8..7)
                quant1 = std::clamp(quant1, int8_t(-8), int8_t(7));
                quant2 = std::clamp(quant2, int8_t(-8), int8_t(7));
    
                // Pack into byte
                out->values[i] = (quant2 << 4) | (quant1 & 0x0F);
            }
    
            out->padding = 0;  // Clear padding byte
        }
    
        // Optional: Helper to analyze the compression
        static void analyze_compression(const void* data) {
            const packed_data* packed = static_cast<const packed_data*>(data);
            
            uint8_t scale_bits = packed->scale_offset >> 2;
            uint8_t offset_bits = packed->scale_offset & 0x3;
            
            float scale = decode_scale(scale_bits);
            float offset = decode_offset(offset_bits);
    
            std::cout << "Compression Analysis:\n"
                      << "  Scale: " << scale << " (encoded: " << (int)scale_bits << ")\n"
                      << "  Offset: " << offset << " (encoded: " << (int)offset_bits << ")\n"
                      << "  Packed values (hex):";
            
            for (int i = 0; i < 8; ++i) {
                std::cout << " " << std::hex << std::setw(2) << std::setfill('0') 
                          << (int)packed->values[i];
            }
            std::cout << std::dec << "\n";
    
            // Show reconstructed values
            std::cout << "  Reconstructed values:";
            for (int i = 0; i < 8; ++i) {
                uint8_t byte = packed->values[i];
                int8_t lo_val = (byte & 0x0F);
                int8_t hi_val = (byte >> 4) & 0x0F;
                
                // Sign extend
                lo_val = (lo_val & 0x08) ? (lo_val | 0xF0) : lo_val;
                hi_val = (hi_val & 0x08) ? (hi_val | 0xF0) : hi_val;
                
                float val1 = lo_val * scale + offset;
                float val2 = hi_val * scale + offset;
                
                std::cout << " " << val1 << " " << val2;
            }
            std::cout << "\n";
        }
    };
    
    // Example usage:
    void test_scaled_int4() {
        // Create some test data
        bundle<float, 16> test_data = bundle<float, 16>::broadcast(0.0f);
        alignas(16) float test_values[16] = {
            1.5f, 2.7f, -1.2f, 4.8f, 
            -3.1f, 0.9f, 2.2f, -2.5f,
            1.8f, -1.7f, 3.3f, -0.8f,
            2.1f, -2.9f, 4.1f, -1.4f
        };
        test_data = bundle<float, 16>::load(test_values);
    
        // Compress
        alignas(16) scaled_int4_codec::packed_data compressed;
        test_data.store<scaled_int4_codec>(&compressed);
    
        // Analyze compression
        std::cout << "Original values:";
        for (int i = 0; i < 16; ++i) {
            std::cout << " " << test_values[i];
        }
        std::cout << "\n\n";
    
        scaled_int4_codec::analyze_compression(&compressed);
    
        // Decompress
        bundle<float, 16> decompressed = bundle<float, 16>::load<scaled_int4_codec>(&compressed);
    
        // Compare
        std::cout << "\nDecompressed values:";
        alignas(16) float final_values[16];
        decompressed.store(final_values);
        for (int i = 0; i < 16; ++i) {
            std::cout << " " << final_values[i];
        }
        std::cout << "\n";
    
        // Calculate error
        float max_error = 0.0f;
        float avg_error = 0.0f;
        for (int i = 0; i < 16; ++i) {
            float error = std::abs(test_values[i] - final_values[i]);
            max_error = std::max(max_error, error);
            avg_error += error;
        }
        avg_error /= 16.0f;
    
        std::cout << "\nCompression stats:\n"
                  << "  Max error: " << max_error << "\n"
                  << "  Avg error: " << avg_error << "\n"
                  << "  Compression ratio: " << (sizeof(test_values) / sizeof(compressed)) << ":1\n";
    }
    

    Key features of this implementation:

    1. Uses a logarithmic scale for better dynamic range coverage
    2. Intelligent offset selection to minimize quantization error
    3. Efficient packing/unpacking using bit operations
    4. Includes analysis tools to understand the compression
    5. Handles sign extension correctly
    6. Uses NEON intrinsics for vector operations
    7. Maintains alignment requirements
    8. Provides detailed error analysis

    The compression achieves a 6.4:1 ratio (64 bytes → 10 bytes) while maintaining reasonable accuracy for many types of data.

    The scale is logarithmic (2^0 to 2^10) to handle both small and large values efficiently. The offset helps center the data to minimize quantization error.

    Would you like me to explain any part in more detail or show additional examples with different types of data?

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.