the decryption function? its failing at "if (EVPDecryptFinalex(ctx, decryptedData.data()…
- User
the decryption function? its failing at "if (EVP_DecryptFinal_ex(ctx, decryptedData.data() + outLen1, &outLen2) != 1)"
i can see that the decryptedData contains the data decrypted correctly, just its size that is wrong fix the decryption function and dont omit anything
inline bool weakEncrypt2(QByteArray& data) {
VM_S_START // C898if (data.isEmpty()) return false; // Generate the password unsigned char random_byte; RAND_bytes(&random_byte, 1); int passwordSize = 9 + (random_byte % 3) * 3; // Random size of 9, 12, or 15 std::vector<unsigned char> password(passwordSize); RAND_bytes(password.data(), passwordSize); // Generate a random salt RAND_bytes(&random_byte, 1); int saltSize = 9 + (random_byte % 3) * 3; // Random size of 9, 12, or 15 std::vector<unsigned char> salt(saltSize); RAND_bytes(salt.data(), saltSize); // Encode the salt as base64 BIO* bio = BIO_push(BIO_new(BIO_f_base64()), BIO_new(BIO_s_mem())); BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); BIO_write(bio, salt.data(), saltSize); BIO_flush(bio); BUF_MEM* bptr; BIO_get_mem_ptr(bio, &bptr); std::vector<unsigned char> b64Salt(bptr->length); memcpy(b64Salt.data(), bptr->data, bptr->length); OPENSSL_cleanse(bptr->data, bptr->length); OPENSSL_cleanse(bptr, sizeof(*bptr)); BIO_free_all(bio); // SHA256 the salt and password int b64SaltSize = b64Salt.size(); unsigned char keyHash[SHA256_DIGEST_LENGTH]; unsigned char ivHash[SHA256_DIGEST_LENGTH]; SHA256(password.data(), passwordSize, keyHash); SHA256(b64Salt.data(), b64SaltSize, ivHash); std::vector<unsigned char> hashedKey(16); std::vector<unsigned char> hashedIv(16); bio = BIO_push(BIO_new(BIO_f_base64()), BIO_new(BIO_s_mem())); BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); if (BIO_write(bio, keyHash, SHA256_DIGEST_LENGTH) != SHA256_DIGEST_LENGTH || BIO_flush(bio) != 1) { BIO_free_all(bio); return false; } BIO_get_mem_ptr(bio, &bptr); std::memcpy(hashedKey.data(), bptr->data, 16); BIO_reset(bio); if (BIO_write(bio, ivHash, SHA256_DIGEST_LENGTH) != SHA256_DIGEST_LENGTH || BIO_flush(bio) != 1) { BIO_free_all(bio); return false; } BIO_get_mem_ptr(bio, &bptr); std::memcpy(hashedIv.data(), bptr->data, 16); BIO_free_all(bio); OPENSSL_cleanse(keyHash, sizeof(keyHash)); OPENSSL_cleanse(ivHash, sizeof(ivHash)); std::vector<unsigned char> cipherBuf(data.size() + EVP_MAX_BLOCK_LENGTH); const EVP_CIPHER* cipherType = EVP_aes_128_cbc(); EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); if (!ctx) return false; if (EVP_EncryptInit_ex(ctx, cipherType, NULL, hashedKey.data(), hashedIv.data()) != 1) { OPENSSL_cleanse(data.data(), data.size()); data.clear(); EVP_CIPHER_CTX_free(ctx); return false; } OPENSSL_cleanse(hashedKey.data(), hashedKey.size()); OPENSSL_cleanse(hashedIv.data(), hashedIv.size()); int outLength1 = 0; if (EVP_EncryptUpdate(ctx, cipherBuf.data(), &outLength1, reinterpret_cast<unsigned char*>(data.data()), data.size()) != 1) { OPENSSL_cleanse(data.data(), data.size()); data.clear(); EVP_CIPHER_CTX_free(ctx); return false; } int outLength2 = 0; if (EVP_EncryptFinal_ex(ctx, cipherBuf.data() + outLength1, &outLength2) != 1) { OPENSSL_cleanse(data.data(), data.size()); data.clear(); EVP_CIPHER_CTX_free(ctx); return false; } EVP_CIPHER_CTX_free(ctx); OPENSSL_cleanse(data.data(), data.size()); data.clear(); // Generate random string to prepend unsigned char randomByte; RAND_bytes(&randomByte, 1); int prependStrSize = 4 + (randomByte % 7); // Random size between 4 and 10 std::vector<unsigned char> prependStr(prependStrSize); RAND_bytes(prependStr.data(), prependStrSize); data.append(prependStr); data.append(password); data.append(salt); data.append(cipherBuf); VM_S_END // C898 return true;}
inline bool weakDecrypt2(QByteArray& data) { if (data.isEmpty()) return false;
const int possibleSizes[] = { 9, 12, 15 }; for (int prependSize = 4; prependSize <= 10; ++prependSize) { for (int passwordSize : possibleSizes) { for (int saltSize : possibleSizes) { if (data.size() < prependSize + passwordSize + saltSize) continue; std::vector<unsigned char> password(passwordSize); std::vector<unsigned char> salt(saltSize); std::memcpy(password.data(), data.data() + prependSize, passwordSize); std::memcpy(salt.data(), data.data() + prependSize + passwordSize, saltSize); // Base64 encode the salt BIO* bio = BIO_push(BIO_new(BIO_f_base64()), BIO_new(BIO_s_mem())); BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); BIO_write(bio, salt.data(), saltSize); BIO_flush(bio); BUF_MEM* bptr; BIO_get_mem_ptr(bio, &bptr); int b64SaltSize = bptr->length; std::vector<unsigned char> b64Salt(b64SaltSize); std::memcpy(b64Salt.data(), bptr->data, bptr->length); BIO_free_all(bio); // SHA256 the password and salt unsigned char keyHash[SHA256_DIGEST_LENGTH]; unsigned char ivHash[SHA256_DIGEST_LENGTH]; SHA256(password.data(), passwordSize, keyHash); SHA256(b64Salt.data(), b64SaltSize, ivHash); std::vector<unsigned char> hashedKey(16); std::vector<unsigned char> hashedIv(16); bio = BIO_push(BIO_new(BIO_f_base64()), BIO_new(BIO_s_mem())); BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); BIO_write(bio, keyHash, SHA256_DIGEST_LENGTH); BIO_flush(bio); BIO_get_mem_ptr(bio, &bptr); std::memcpy(hashedKey.data(), bptr->data, 16); BIO_reset(bio); BIO_write(bio, ivHash, SHA256_DIGEST_LENGTH); BIO_flush(bio); BIO_get_mem_ptr(bio, &bptr); std::memcpy(hashedIv.data(), bptr->data, 16); BIO_free_all(bio); // Attempt decryption const EVP_CIPHER* cipherType = EVP_aes_128_cbc(); EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); if (!ctx) continue; if (EVP_DecryptInit_ex(ctx, cipherType, NULL, hashedKey.data(), hashedIv.data()) != 1) { EVP_CIPHER_CTX_free(ctx); continue; } std::vector<unsigned char> decryptedData(data.size()); int outLen1 = 0; if (EVP_DecryptUpdate(ctx, decryptedData.data(), &outLen1, reinterpret_cast<const unsigned char*>(data.data()) + prependSize + passwordSize + saltSize, data.size() - prependSize - passwordSize - saltSize) != 1) { EVP_CIPHER_CTX_free(ctx); continue; } int outLen2 = 0; if (EVP_DecryptFinal_ex(ctx, decryptedData.data() + outLen1, &outLen2) != 1) { EVP_CIPHER_CTX_free(ctx); continue; } EVP_CIPHER_CTX_free(ctx); // Decryption successful decryptedData.resize(outLen1 + outLen2); data = QByteArray(reinterpret_cast<char*>(decryptedData.data()), decryptedData.size()); OPENSSL_cleanse(password.data(), password.size()); OPENSSL_cleanse(salt.data(), salt.size()); OPENSSL_cleanse(b64Salt.data(), b64Salt.size()); OPENSSL_cleanse(decryptedData.data(), decryptedData.size()); return true; } } } // Decryption failed return false;}
- Assistant
The issue you're encountering with the
weakDecrypt2function failing at the line:if (EVP_DecryptFinal_ex(ctx, decryptedData.data() + outLen1, &outLen2) != 1)is primarily due to how the encrypted data (
cipherBuf) is handled in theweakEncrypt2function. Specifically, the encryption function appends the entirecipherBufbuffer to thedata, including any unused bytes beyond the actual encrypted data (outLen1 + outLen2). This results in extra bytes being present in the ciphertext, which in turn causesEVP_DecryptFinal_exto fail during decryption due to padding errors.To resolve this, you need to ensure that only the actual encrypted bytes are appended to the
datain theweakEncrypt2function. Additionally, it's good practice to handle all OpenSSL operations carefully to avoid memory leaks and ensure data integrity.Below are the corrected versions of both
weakEncrypt2andweakDecrypt2functions with detailed explanations:
Corrected
weakEncrypt2Function#include <QByteArray> #include <vector> #include <openssl/evp.h> #include <openssl/rand.h> #include <openssl/sha.h> #include <openssl/bio.h> #include <openssl/buffer.h> #include <cstring> inline bool weakEncrypt2(QByteArray& data) { VM_S_START // C898 if (data.isEmpty()) return false; // Generate the password unsigned char random_byte; if (RAND_bytes(&random_byte, 1) != 1) return false; int passwordSize = 9 + (random_byte % 3) * 3; // Random size of 9, 12, or 15 std::vector<unsigned char> password(passwordSize); if (RAND_bytes(password.data(), passwordSize) != 1) return false; // Generate a random salt if (RAND_bytes(&random_byte, 1) != 1) return false; int saltSize = 9 + (random_byte % 3) * 3; // Random size of 9, 12, or 15 std::vector<unsigned char> salt(saltSize); if (RAND_bytes(salt.data(), saltSize) != 1) return false; // Encode the salt as base64 BIO* bio = BIO_new(BIO_s_mem()); BIO* b64 = BIO_new(BIO_f_base64()); if (!bio || !b64) { BIO_free_all(bio); BIO_free_all(b64); return false; } bio = BIO_push(b64, bio); BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); if (BIO_write(bio, salt.data(), saltSize) != saltSize) { BIO_free_all(bio); return false; } if (BIO_flush(bio) != 1) { BIO_free_all(bio); return false; } BUF_MEM* bptr; BIO_get_mem_ptr(bio, &bptr); std::vector<unsigned char> b64Salt(bptr->length); memcpy(b64Salt.data(), bptr->data, bptr->length); OPENSSL_cleanse(bptr->data, bptr->length); OPENSSL_cleanse(bptr, sizeof(*bptr)); BIO_free_all(bio); // SHA256 the salt and password unsigned char keyHash[SHA256_DIGEST_LENGTH]; unsigned char ivHash[SHA256_DIGEST_LENGTH]; SHA256(password.data(), passwordSize, keyHash); SHA256(b64Salt.data(), b64Salt.size(), ivHash); std::vector<unsigned char> hashedKey(16); std::vector<unsigned char> hashedIv(16); // Base64 encode keyHash bio = BIO_new(BIO_s_mem()); b64 = BIO_new(BIO_f_base64()); if (!bio || !b64) { BIO_free_all(bio); BIO_free_all(b64); return false; } bio = BIO_push(b64, bio); BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); if (BIO_write(bio, keyHash, SHA256_DIGEST_LENGTH) != SHA256_DIGEST_LENGTH) { BIO_free_all(bio); return false; } if (BIO_flush(bio) != 1) { BIO_free_all(bio); return false; } BIO_get_mem_ptr(bio, &bptr); if (bptr->length < 16) { BIO_free_all(bio); return false; } std::memcpy(hashedKey.data(), bptr->data, 16); BIO_free_all(bio); // Base64 encode ivHash bio = BIO_new(BIO_s_mem()); b64 = BIO_new(BIO_f_base64()); if (!bio || !b64) { BIO_free_all(bio); BIO_free_all(b64); return false; } bio = BIO_push(b64, bio); BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); if (BIO_write(bio, ivHash, SHA256_DIGEST_LENGTH) != SHA256_DIGEST_LENGTH) { BIO_free_all(bio); return false; } if (BIO_flush(bio) != 1) { BIO_free_all(bio); return false; } BIO_get_mem_ptr(bio, &bptr); if (bptr->length < 16) { BIO_free_all(bio); return false; } std::memcpy(hashedIv.data(), bptr->data, 16); BIO_free_all(bio); OPENSSL_cleanse(keyHash, sizeof(keyHash)); OPENSSL_cleanse(ivHash, sizeof(ivHash)); // Initialize encryption std::vector<unsigned char> cipherBuf(data.size() + EVP_MAX_BLOCK_LENGTH); const EVP_CIPHER* cipherType = EVP_aes_128_cbc(); EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); if (!ctx) return false; if (EVP_EncryptInit_ex(ctx, cipherType, NULL, hashedKey.data(), hashedIv.data()) != 1) { OPENSSL_cleanse(data.data(), data.size()); data.clear(); EVP_CIPHER_CTX_free(ctx); return false; } OPENSSL_cleanse(hashedKey.data(), hashedKey.size()); OPENSSL_cleanse(hashedIv.data(), hashedIv.size()); int outLength1 = 0; if (EVP_EncryptUpdate(ctx, cipherBuf.data(), &outLength1, reinterpret_cast<const unsigned char*>(data.data()), data.size()) != 1) { OPENSSL_cleanse(data.data(), data.size()); data.clear(); EVP_CIPHER_CTX_free(ctx); return false; } int outLength2 = 0; if (EVP_EncryptFinal_ex(ctx, cipherBuf.data() + outLength1, &outLength2) != 1) { OPENSSL_cleanse(data.data(), data.size()); data.clear(); EVP_CIPHER_CTX_free(ctx); return false; } EVP_CIPHER_CTX_free(ctx); // Resize cipherBuf to actual encrypted size cipherBuf.resize(outLength1 + outLength2); OPENSSL_cleanse(data.data(), data.size()); data.clear(); // Generate random string to prepend unsigned char randomByte; if (RAND_bytes(&randomByte, 1) != 1) return false; int prependStrSize = 4 + (randomByte % 7); // Random size between 4 and 10 std::vector<unsigned char> prependStr(prependStrSize); if (RAND_bytes(prependStr.data(), prependStrSize) != 1) return false; // Append prependStr, password, salt, and cipherBuf to data data.append(reinterpret_cast<char*>(prependStr.data()), prependStrSize); data.append(reinterpret_cast<char*>(password.data()), passwordSize); data.append(reinterpret_cast<char*>(salt.data()), saltSize); data.append(reinterpret_cast<char*>(cipherBuf.data()), cipherBuf.size()); VM_S_END // C898 return true; }Key Changes and Explanations:
-
Resizing
cipherBufAfter Encryption:After performing
EVP_EncryptFinal_ex, onlyoutLength1 + outLength2bytes ofcipherBufcontain valid encrypted data. The rest of the buffer may contain uninitialized or redundant bytes. To ensure only the actual encrypted data is appended todata, resizecipherBufaccordingly:cipherBuf.resize(outLength1 + outLength2); -
Robust Error Handling:
Added checks after each OpenSSL function call to ensure they succeed. If any step fails, the function cleans up appropriately and returns
false. -
Proper Memory Management:
Ensured all BIO structures are freed after use to prevent memory leaks.
-
Consistent Base64 Encoding:
Fixed the base64 encoding steps to ensure correct handling of the
hashedKeyandhashedIv. -
Security Enhancements:
While the function is labeled as
weak, it's crucial to use secure practices wherever possible. Ensure that sensitive data is properly cleansed and handled.
Corrected
weakDecrypt2Function#include <QByteArray> #include <vector> #include <openssl/evp.h> #include <openssl/rand.h> #include <openssl/sha.h> #include <openssl/bio.h> #include <openssl/buffer.h> #include <cstring> inline bool weakDecrypt2(QByteArray& data) { if (data.isEmpty()) return false; const int possibleSizes[] = { 9, 12, 15 }; for (int prependSize = 4; prependSize <= 10; ++prependSize) { for (int passwordSize : possibleSizes) { for (int saltSize : possibleSizes) { if (data.size() < prependSize + passwordSize + saltSize) continue; std::vector<unsigned char> password(passwordSize); std::vector<unsigned char> salt(saltSize); std::memcpy(password.data(), data.data() + prependSize, passwordSize); std::memcpy(salt.data(), data.data() + prependSize + passwordSize, saltSize); // Base64 encode the salt BIO* bio = BIO_new(BIO_s_mem()); BIO* b64 = BIO_new(BIO_f_base64()); if (!bio || !b64) { BIO_free_all(bio); BIO_free_all(b64); continue; } bio = BIO_push(b64, bio); BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); if (BIO_write(bio, salt.data(), saltSize) != saltSize) { BIO_free_all(bio); continue; } if (BIO_flush(bio) != 1) { BIO_free_all(bio); continue; } BUF_MEM* bptr; BIO_get_mem_ptr(bio, &bptr); std::vector<unsigned char> b64Salt(bptr->length); memcpy(b64Salt.data(), bptr->data, bptr->length); BIO_free_all(bio); // SHA256 the password and salt unsigned char keyHash[SHA256_DIGEST_LENGTH]; unsigned char ivHash[SHA256_DIGEST_LENGTH]; SHA256(password.data(), passwordSize, keyHash); SHA256(b64Salt.data(), b64Salt.size(), ivHash); std::vector<unsigned char> hashedKey(16); std::vector<unsigned char> hashedIv(16); // Base64 encode keyHash bio = BIO_new(BIO_s_mem()); b64 = BIO_new(BIO_f_base64()); if (!bio || !b64) { BIO_free_all(bio); BIO_free_all(b64); continue; } bio = BIO_push(b64, bio); BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); if (BIO_write(bio, keyHash, SHA256_DIGEST_LENGTH) != SHA256_DIGEST_LENGTH) { BIO_free_all(bio); continue; } if (BIO_flush(bio) != 1) { BIO_free_all(bio); continue; } BIO_get_mem_ptr(bio, &bptr); if (bptr->length < 16) { BIO_free_all(bio); continue; } std::memcpy(hashedKey.data(), bptr->data, 16); BIO_free_all(bio); // Base64 encode ivHash bio = BIO_new(BIO_s_mem()); b64 = BIO_new(BIO_f_base64()); if (!bio || !b64) { BIO_free_all(bio); BIO_free_all(b64); continue; } bio = BIO_push(b64, bio); BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); if (BIO_write(bio, ivHash, SHA256_DIGEST_LENGTH) != SHA256_DIGEST_LENGTH) { BIO_free_all(bio); continue; } if (BIO_flush(bio) != 1) { BIO_free_all(bio); continue; } BIO_get_mem_ptr(bio, &bptr); if (bptr->length < 16) { BIO_free_all(bio); continue; } std::memcpy(hashedIv.data(), bptr->data, 16); BIO_free_all(bio); // Initialize decryption const EVP_CIPHER* cipherType = EVP_aes_128_cbc(); EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); if (!ctx) { continue; } if (EVP_DecryptInit_ex(ctx, cipherType, NULL, hashedKey.data(), hashedIv.data()) != 1) { EVP_CIPHER_CTX_free(ctx); continue; } // Determine ciphertext size int ciphertextSize = data.size() - prependSize - passwordSize - saltSize; if (ciphertextSize <= 0) { EVP_CIPHER_CTX_free(ctx); continue; } std::vector<unsigned char> decryptedData(ciphertextSize + EVP_MAX_BLOCK_LENGTH); int outLen1 = 0; if (EVP_DecryptUpdate(ctx, decryptedData.data(), &outLen1, reinterpret_cast<const unsigned char*>(data.data()) + prependSize + passwordSize + saltSize, ciphertextSize) != 1) { EVP_CIPHER_CTX_free(ctx); continue; } int outLen2 = 0; if (EVP_DecryptFinal_ex(ctx, decryptedData.data() + outLen1, &outLen2) != 1) { // Decryption failed, possibly due to incorrect padding EVP_CIPHER_CTX_free(ctx); continue; } EVP_CIPHER_CTX_free(ctx); // Resize decryptedData to actual decrypted size decryptedData.resize(outLen1 + outLen2); data = QByteArray(reinterpret_cast<char*>(decryptedData.data()), decryptedData.size()); OPENSSL_cleanse(password.data(), password.size()); OPENSSL_cleanse(salt.data(), salt.size()); OPENSSL_cleanse(b64Salt.data(), b64Salt.size()); OPENSSL_cleanse(decryptedData.data(), decryptedData.size()); return true; } } } // Decryption failed return false; }Key Changes and Explanations:
-
Correct Handling of Ciphertext Size:
-
Issue: In the original
weakEncrypt2, the entirecipherBuf(including unused bytes) was appended todata. This causedweakDecrypt2to attempt decrypting more bytes than necessary, leading to padding errors andEVP_DecryptFinal_exfailing. -
Fix: By resizing
cipherBufin theweakEncrypt2function to include only the actual encrypted bytes, we ensure thatweakDecrypt2correctly calculates theciphertextSizeasdata.size() - prependSize - passwordSize - saltSize.
-
-
Robust Error Handling:
Added checks after each OpenSSL function call to ensure they succeed. If any step fails, the function cleans up appropriately and continues to the next iteration.
-
Proper Memory Management:
Ensured all BIO structures are freed after use to prevent memory leaks.
-
Consistent Base64 Encoding:
Fixed the base64 encoding steps to ensure correct handling of the
hashedKeyandhashedIv. -
Security Enhancements:
While the functions are labeled as
weak, it's crucial to handle sensitive data securely. Ensure that sensitive buffers are cleansed after use. -
Validation of Ciphertext Size:
Before attempting decryption, verify that
ciphertextSizeis positive to prevent invalid memory access.
Summary
The primary issue causing
EVP_DecryptFinal_exto fail was the improper handling of the encrypted buffer (cipherBuf) in theweakEncrypt2function. By resizingcipherBufto include only the actual encrypted bytes (outLength1 + outLength2), you ensure that the decryption function processes the correct amount of data. Additionally, robust error handling and proper memory management are essential to maintain the stability and security of your encryption and decryption processes.Recommendations:
-
Use Modern Encryption Standards: While AES-128-CBC is widely used, consider using authenticated encryption modes like AES-256-GCM for better security.
-
Avoid "Weak" Encryption Practices: Generating passwords and salts with random sizes and managing keys in this manner can introduce vulnerabilities. It's advisable to follow established cryptographic best practices.
-
Handle Errors Gracefully: Always ensure that cryptographic operations are checked for success and handle failures appropriately to avoid unintended behavior.
-
Secure Memory Handling: Ensure that all sensitive data is properly cleansed from memory immediately after use to prevent potential leaks.
By implementing these corrections and recommendations, your encryption and decryption functions should operate reliably and securely.
-