Super and now implement a format where 16 values are stored as 4 bit, with a shared 6…
- User
Super and now implement a format where 16 values are stored as 4 bit, with a shared 6 bits scale (on a log scale from 1..1024) and a 2 bit offset(-2,-1,+1,+2).
Assistant
I'll create a
scaled_int4_codecthat implements this format. The 16 values will be stored in 10 bytes total:- 1 byte for scale (6 bits) and offset (2 bits)
- 8 bytes for the 16 4-bit values
- 1 byte unused (for alignment)
Here's the implementation:
struct scaled_int4_codec { // Helper functions for scale/offset encoding/decoding static constexpr float decode_scale(uint8_t scale_bits) { // 6 bits give us 64 possible values // We'll use a log scale from 2^0 to 2^10 (1 to 1024) return std::pow(2.0f, (scale_bits * 10.0f) / 63.0f); } static constexpr uint8_t encode_scale(float scale) { // Clamp scale to [1, 1024] scale = std::clamp(scale, 1.0f, 1024.0f); // Convert to 6-bit value float log2_scale = std::log2(scale); return static_cast<uint8_t>((log2_scale * 63.0f) / 10.0f); } static constexpr float decode_offset(uint8_t offset_bits) { // 2 bits encode: -2, -1, +1, +2 static constexpr float offsets[] = {-2.0f, -1.0f, 1.0f, 2.0f}; return offsets[offset_bits]; } static constexpr uint8_t encode_offset(float offset) { // Find closest match to -2, -1, 1, 2 if (offset <= -1.5f) return 0; // -2 else if (offset <= 0.0f) return 1; // -1 else if (offset <= 1.5f) return 2; // +1 else return 3; // +2 } // Structure representing the packed format struct packed_data { uint8_t scale_offset; // 6 bits scale, 2 bits offset uint8_t values[8]; // 16 4-bit values uint8_t padding; // Alignment padding }; static_assert(sizeof(packed_data) == 10, "packed_data must be 10 bytes"); template <typename TargetValueType> static constexpr bundle<TargetValueType, 16> load(const void* src) requires std::is_same_v<TargetValueType, float> { const packed_data* data = static_cast<const packed_data*>(src); // Extract scale and offset uint8_t scale_bits = data->scale_offset >> 2; // Upper 6 bits uint8_t offset_bits = data->scale_offset & 0x3; // Lower 2 bits float scale = decode_scale(scale_bits); float offset = decode_offset(offset_bits); // Create result bundle bundle<float, 16> result; alignas(16) float unpacked[16]; // Unpack each byte into two float values for (size_t i = 0; i < 8; ++i) { uint8_t byte = data->values[i]; // Extract low and high nibbles int8_t lo_val = byte & 0x0F; int8_t hi_val = (byte >> 4) & 0x0F; // Convert to -8..7 range lo_val = (lo_val & 0x08) ? (lo_val | 0xF0) : lo_val; hi_val = (hi_val & 0x08) ? (hi_val | 0xF0) : hi_val; // Apply scale and offset unpacked[i * 2] = static_cast<float>(lo_val) * scale + offset; unpacked[i * 2 + 1] = static_cast<float>(hi_val) * scale + offset; } // Load unpacked values into Neon vector result.data = vld1q_f32_x4(unpacked); return result; } template <typename SourceValueType> static constexpr void store(const bundle<SourceValueType, 16>& bundle_data, void* dst) requires std::is_same_v<SourceValueType, float> { packed_data* out = static_cast<packed_data*>(dst); alignas(16) float values[16]; // Store vector data to array for processing vst1q_f32_x4(values, bundle_data.data); // First pass: find scale and offset float min_val = values[0]; float max_val = values[0]; for (int i = 1; i < 16; ++i) { min_val = std::min(min_val, values[i]); max_val = std::max(max_val, values[i]); } // Calculate optimal scale and offset float data_range = max_val - min_val; float center = (max_val + min_val) / 2.0f; // We need to map to -8..7 range float scale = data_range / 15.0f; // 15 is the range of our int4 (-8..7) // Find best offset from {-2, -1, 1, 2} float best_offset = 1.0f; // Default float min_error = std::numeric_limits<float>::max(); for (float test_offset : {-2.0f, -1.0f, 1.0f, 2.0f}) { float error = std::abs(center - test_offset); if (error < min_error) { min_error = error; best_offset = test_offset; } } // Store scale and offset uint8_t scale_bits = encode_scale(scale); uint8_t offset_bits = encode_offset(best_offset); out->scale_offset = (scale_bits << 2) | offset_bits; // Second pass: quantize values scale = decode_scale(scale_bits); // Use actual encoded scale float offset = decode_offset(offset_bits); // Use actual encoded offset for (size_t i = 0; i < 8; ++i) { // Process two values at a time float val1 = values[i * 2]; float val2 = values[i * 2 + 1]; // Remove offset and scale int8_t quant1 = static_cast<int8_t>(std::round((val1 - offset) / scale)); int8_t quant2 = static_cast<int8_t>(std::round((val2 - offset) / scale)); // Clamp to int4 range (-8..7) quant1 = std::clamp(quant1, int8_t(-8), int8_t(7)); quant2 = std::clamp(quant2, int8_t(-8), int8_t(7)); // Pack into byte out->values[i] = (quant2 << 4) | (quant1 & 0x0F); } out->padding = 0; // Clear padding byte } // Optional: Helper to analyze the compression static void analyze_compression(const void* data) { const packed_data* packed = static_cast<const packed_data*>(data); uint8_t scale_bits = packed->scale_offset >> 2; uint8_t offset_bits = packed->scale_offset & 0x3; float scale = decode_scale(scale_bits); float offset = decode_offset(offset_bits); std::cout << "Compression Analysis:\n" << " Scale: " << scale << " (encoded: " << (int)scale_bits << ")\n" << " Offset: " << offset << " (encoded: " << (int)offset_bits << ")\n" << " Packed values (hex):"; for (int i = 0; i < 8; ++i) { std::cout << " " << std::hex << std::setw(2) << std::setfill('0') << (int)packed->values[i]; } std::cout << std::dec << "\n"; // Show reconstructed values std::cout << " Reconstructed values:"; for (int i = 0; i < 8; ++i) { uint8_t byte = packed->values[i]; int8_t lo_val = (byte & 0x0F); int8_t hi_val = (byte >> 4) & 0x0F; // Sign extend lo_val = (lo_val & 0x08) ? (lo_val | 0xF0) : lo_val; hi_val = (hi_val & 0x08) ? (hi_val | 0xF0) : hi_val; float val1 = lo_val * scale + offset; float val2 = hi_val * scale + offset; std::cout << " " << val1 << " " << val2; } std::cout << "\n"; } }; // Example usage: void test_scaled_int4() { // Create some test data bundle<float, 16> test_data = bundle<float, 16>::broadcast(0.0f); alignas(16) float test_values[16] = { 1.5f, 2.7f, -1.2f, 4.8f, -3.1f, 0.9f, 2.2f, -2.5f, 1.8f, -1.7f, 3.3f, -0.8f, 2.1f, -2.9f, 4.1f, -1.4f }; test_data = bundle<float, 16>::load(test_values); // Compress alignas(16) scaled_int4_codec::packed_data compressed; test_data.store<scaled_int4_codec>(&compressed); // Analyze compression std::cout << "Original values:"; for (int i = 0; i < 16; ++i) { std::cout << " " << test_values[i]; } std::cout << "\n\n"; scaled_int4_codec::analyze_compression(&compressed); // Decompress bundle<float, 16> decompressed = bundle<float, 16>::load<scaled_int4_codec>(&compressed); // Compare std::cout << "\nDecompressed values:"; alignas(16) float final_values[16]; decompressed.store(final_values); for (int i = 0; i < 16; ++i) { std::cout << " " << final_values[i]; } std::cout << "\n"; // Calculate error float max_error = 0.0f; float avg_error = 0.0f; for (int i = 0; i < 16; ++i) { float error = std::abs(test_values[i] - final_values[i]); max_error = std::max(max_error, error); avg_error += error; } avg_error /= 16.0f; std::cout << "\nCompression stats:\n" << " Max error: " << max_error << "\n" << " Avg error: " << avg_error << "\n" << " Compression ratio: " << (sizeof(test_values) / sizeof(compressed)) << ":1\n"; }Key features of this implementation:
- Uses a logarithmic scale for better dynamic range coverage
- Intelligent offset selection to minimize quantization error
- Efficient packing/unpacking using bit operations
- Includes analysis tools to understand the compression
- Handles sign extension correctly
- Uses NEON intrinsics for vector operations
- Maintains alignment requirements
- Provides detailed error analysis
The compression achieves a 6.4:1 ratio (64 bytes → 10 bytes) while maintaining reasonable accuracy for many types of data.
The scale is logarithmic (2^0 to 2^10) to handle both small and large values efficiently. The offset helps center the data to minimize quantization error.
Would you like me to explain any part in more detail or show additional examples with different types of data?
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.