From be2cdc7dfe792cd2414b2c01ab0ad7087b7f98dc Mon Sep 17 00:00:00 2001 From: vit9696 Date: Thu, 3 Jan 2019 22:53:31 +0300 Subject: [PATCH] Implement custom LZMA dictionary size for NE --- common/LZMA/LzmaCompress.c | 6 +- common/LZMA/LzmaCompress.h | 5 +- common/basetypes.h | 47 ++++++------- common/ffsparser.cpp | 132 +++++++++++++++++++------------------ common/parsingdata.h | 10 ++- common/treemodel.cpp | 2 + common/utility.cpp | 22 ++++--- common/utility.h | 4 +- 8 files changed, 124 insertions(+), 104 deletions(-) diff --git a/common/LZMA/LzmaCompress.c b/common/LZMA/LzmaCompress.c index 25ca4e3..25b086b 100644 --- a/common/LZMA/LzmaCompress.c +++ b/common/LZMA/LzmaCompress.c @@ -64,7 +64,8 @@ LzmaCompress ( CONST UINT8 *Source, UINT32 SourceSize, UINT8 *Destination, - UINT32 *DestinationSize + UINT32 *DestinationSize, + UINT32 DictionarySize ) { SRes LzmaResult; @@ -79,8 +80,7 @@ LzmaCompress ( } LzmaEncProps_Init(&props); - // TODO: need to detect this instead of hardcoding - props.dictSize = LZMA_DICTIONARY_SIZE; + props.dictSize = DictionarySize; props.level = 9; props.fb = 273; diff --git a/common/LZMA/LzmaCompress.h b/common/LZMA/LzmaCompress.h index 94ec9ed..08bbe74 100644 --- a/common/LZMA/LzmaCompress.h +++ b/common/LZMA/LzmaCompress.h @@ -21,7 +21,7 @@ extern "C" { #endif -#define LZMA_DICTIONARY_SIZE 0x800000 +#define DEFAULT_LZMA_DICTIONARY_SIZE 0x800000 #define _LZMA_SIZE_OPT USTATUS @@ -30,7 +30,8 @@ extern "C" { const UINT8 *Source, UINT32 SourceSize, UINT8 *Destination, - UINT32 *DestinationSize + UINT32 *DestinationSize, + UINT32 DictionarySize ); #ifdef __cplusplus diff --git a/common/basetypes.h b/common/basetypes.h index 89ead20..de4ae3b 100644 --- a/common/basetypes.h +++ b/common/basetypes.h @@ -45,31 +45,32 @@ typedef size_t USTATUS; #define U_CUSTOMIZED_COMPRESSION_FAILED 23 #define U_STANDARD_DECOMPRESSION_FAILED 24 #define U_CUSTOMIZED_DECOMPRESSION_FAILED 25 -#define U_UNKNOWN_COMPRESSION_TYPE 26 -#define U_DEPEX_PARSE_FAILED 27 -#define U_UNKNOWN_EXTRACT_MODE 28 -#define U_UNKNOWN_REPLACE_MODE 29 -#define U_UNKNOWN_IMAGE_TYPE 30 -#define U_UNKNOWN_PE_OPTIONAL_HEADER_TYPE 31 -#define U_UNKNOWN_RELOCATION_TYPE 32 -#define U_DIR_ALREADY_EXIST 33 -#define U_DIR_CREATE 34 -#define U_DIR_CHANGE 35 -#define U_TRUNCATED_IMAGE 36 -#define U_INVALID_CAPSULE 37 -#define U_STORES_NOT_FOUND 38 -#define U_INVALID_IMAGE 39 -#define U_INVALID_RAW_AREA 40 -#define U_INVALID_FIT 41 -#define U_INVALID_MICROCODE 42 -#define U_INVALID_ACM 43 -#define U_INVALID_BG_KEY_MANIFEST 44 -#define U_INVALID_BG_BOOT_POLICY 45 -#define U_INVALID_TXT_CONF 46 -#define U_ELEMENTS_NOT_FOUND 47 +#define U_GZIP_DECOMPRESSION_FAILED 26 +#define U_UNKNOWN_COMPRESSION_TYPE 27 +#define U_DEPEX_PARSE_FAILED 28 +#define U_UNKNOWN_EXTRACT_MODE 29 +#define U_UNKNOWN_REPLACE_MODE 30 +#define U_UNKNOWN_IMAGE_TYPE 31 +#define U_UNKNOWN_PE_OPTIONAL_HEADER_TYPE 32 +#define U_UNKNOWN_RELOCATION_TYPE 33 +#define U_DIR_ALREADY_EXIST 34 +#define U_DIR_CREATE 35 +#define U_DIR_CHANGE 36 +#define U_TRUNCATED_IMAGE 37 +#define U_INVALID_CAPSULE 38 +#define U_STORES_NOT_FOUND 39 +#define U_INVALID_IMAGE 40 +#define U_INVALID_RAW_AREA 41 +#define U_INVALID_FIT 42 +#define U_INVALID_MICROCODE 43 +#define U_INVALID_ACM 44 +#define U_INVALID_BG_KEY_MANIFEST 45 +#define U_INVALID_BG_BOOT_POLICY 46 +#define U_INVALID_TXT_CONF 47 +#define U_ELEMENTS_NOT_FOUND 48 #define U_NOT_IMPLEMENTED 0xFF -// UDK porting definitions +// EDK2 porting definitions typedef uint8_t BOOLEAN; typedef int8_t INT8; typedef uint8_t UINT8; diff --git a/common/ffsparser.cpp b/common/ffsparser.cpp index 8c64760..44a0700 100644 --- a/common/ffsparser.cpp +++ b/common/ffsparser.cpp @@ -1285,81 +1285,75 @@ USTATUS FfsParser::parseVolumeBody(const UModelIndex & index) while (fileOffset < volumeBodySize) { UINT32 fileSize = getFileSize(volumeBody, fileOffset, ffsVersion); - // Check file size - if (fileSize < sizeof(EFI_FFS_FILE_HEADER) || fileSize > volumeBodySize - fileOffset) { - // Check that we are at the empty space - UByteArray header = volumeBody.mid(fileOffset, sizeof(EFI_FFS_FILE_HEADER)); - if (header.count(emptyByte) == header.size()) { //Empty space - // Check volume usedSpace entry to be valid - if (usedSpace > 0 && usedSpace == fileOffset + volumeHeaderSize) { - if (model->hasEmptyParsingData(index) == false) { - UByteArray data = model->parsingData(index); - VOLUME_PARSING_DATA* pdata = (VOLUME_PARSING_DATA*)data.data(); - pdata->hasValidUsedSpace = TRUE; - model->setParsingData(index, data); - model->setText(index, model->text(index) + "UsedSpace "); - } + + // Check that we are at the empty space + UByteArray header = volumeBody.mid(fileOffset, std::min(sizeof(EFI_FFS_FILE_HEADER), (size_t)volumeBodySize - fileOffset)); + if (header.count(emptyByte) == header.size()) { //Empty space + // Check volume usedSpace entry to be valid + if (usedSpace > 0 && usedSpace == fileOffset + volumeHeaderSize) { + if (model->hasEmptyParsingData(index) == false) { + UByteArray data = model->parsingData(index); + VOLUME_PARSING_DATA* pdata = (VOLUME_PARSING_DATA*)data.data(); + pdata->hasValidUsedSpace = TRUE; + model->setParsingData(index, data); + model->setText(index, model->text(index) + "UsedSpace "); } - - // Check free space to be actually free - UByteArray freeSpace = volumeBody.mid(fileOffset); - if (freeSpace.count(emptyByte) != freeSpace.size()) { - // Search for the first non-empty byte - UINT32 i; - UINT32 size = freeSpace.size(); - const UINT8* current = (UINT8*)freeSpace.constData(); - for (i = 0; i < size; i++) { - if (*current++ != emptyByte) - break; - } + } - // Align found index to file alignment - // It must be possible because minimum 16 bytes of empty were found before - if (i != ALIGN8(i)) { - i = ALIGN8(i) - 8; - } - - // Add all bytes before as free space - if (i > 0) { - UByteArray free = freeSpace.left(i); - - // Get info - UString info = usprintf("Full size: %Xh (%u)", free.size(), free.size()); - - // Add free space item - model->addItem(model->offset(index) + volumeHeaderSize + fileOffset, Types::FreeSpace, 0, UString("Volume free space"), UString(), info, UByteArray(), free, UByteArray(), Movable, index); - } - - // Parse non-UEFI data - parseVolumeNonUefiData(freeSpace.mid(i), volumeHeaderSize + fileOffset + i, index); + // Check free space to be actually free + UByteArray freeSpace = volumeBody.mid(fileOffset); + if (freeSpace.count(emptyByte) != freeSpace.size()) { + // Search for the first non-empty byte + UINT32 i; + UINT32 size = freeSpace.size(); + const UINT8* current = (UINT8*)freeSpace.constData(); + for (i = 0; i < size; i++) { + if (*current++ != emptyByte) + break; } - else { + + // Align found index to file alignment + // It must be possible because minimum 16 bytes of empty were found before + if (i != ALIGN8(i)) { + i = ALIGN8(i) - 8; + } + + // Add all bytes before as free space + if (i > 0) { + UByteArray free = freeSpace.left(i); + // Get info - UString info = usprintf("Full size: %Xh (%u)", freeSpace.size(), freeSpace.size()); + UString info = usprintf("Full size: %Xh (%u)", free.size(), free.size()); // Add free space item - model->addItem(model->offset(index) + volumeHeaderSize + fileOffset, Types::FreeSpace, 0, UString("Volume free space"), UString(), info, UByteArray(), freeSpace, UByteArray(), Movable, index); + model->addItem(model->offset(index) + volumeHeaderSize + fileOffset, Types::FreeSpace, 0, UString("Volume free space"), UString(), info, UByteArray(), free, UByteArray(), Movable, index); } - break; // Exit from parsing loop + + // Parse non-UEFI data + parseVolumeNonUefiData(freeSpace.mid(i), volumeHeaderSize + fileOffset + i, index); } - else { //File space - // Parse non-UEFI data - parseVolumeNonUefiData(volumeBody.mid(fileOffset), volumeHeaderSize + fileOffset, index); - break; // Exit from parsing loop + else { + // Get info + UString info = usprintf("Full size: %Xh (%u)", freeSpace.size(), freeSpace.size()); + + // Add free space item + model->addItem(model->offset(index) + volumeHeaderSize + fileOffset, Types::FreeSpace, 0, UString("Volume free space"), UString(), info, UByteArray(), freeSpace, UByteArray(), Movable, index); } + break; // Exit from parsing loop } - // Get file header - UByteArray file = volumeBody.mid(fileOffset, fileSize); - UByteArray header = file.left(sizeof(EFI_FFS_FILE_HEADER)); - const EFI_FFS_FILE_HEADER* fileHeader = (const EFI_FFS_FILE_HEADER*)header.constData(); - if (ffsVersion == 3 && (fileHeader->Attributes & FFS_ATTRIB_LARGE_FILE)) { - header = file.left(sizeof(EFI_FFS_FILE_HEADER2)); + // We aren't at the end of empty space + // Check that the remaining space can still have a file in it + if (volumeBodySize - fileOffset < sizeof(EFI_FFS_FILE_HEADER) || // Remaining space is smaller than the smallest possible file + volumeBodySize - fileOffset < fileSize) { // Remaining space is smaller than non-empty file size + // Parse non-UEFI data + parseVolumeNonUefiData(volumeBody.mid(fileOffset), volumeHeaderSize + fileOffset, index); + break; // Exit from parsing loop } // Parse current file's header UModelIndex fileIndex; - USTATUS result = parseFileHeader(file, volumeHeaderSize + fileOffset, index, fileIndex); + USTATUS result = parseFileHeader(volumeBody.mid(fileOffset, fileSize), volumeHeaderSize + fileOffset, index, fileIndex); if (result) { msg(usprintf("%s: file header parsing failed with error ", __FUNCTION__) + errorCodeToUString(result), index); } @@ -2462,9 +2456,10 @@ USTATUS FfsParser::parseCompressedSectionBody(const UModelIndex & index) // Decompress section UINT8 algorithm = COMPRESSION_ALGORITHM_NONE; + UINT32 dictionarySize = 0; UByteArray decompressed; UByteArray efiDecompressed; - USTATUS result = decompress(model->body(index), compressionType, algorithm, decompressed, efiDecompressed); + USTATUS result = decompress(model->body(index), compressionType, algorithm, dictionarySize, decompressed, efiDecompressed); if (result) { msg(UString("parseCompressedSectionBody: decompression failed with error ") + errorCodeToUString(result), index); return U_SUCCESS; @@ -2497,10 +2492,14 @@ USTATUS FfsParser::parseCompressedSectionBody(const UModelIndex & index) // Add info model->addInfo(index, UString("\nCompression algorithm: ") + compressionTypeToUString(algorithm)); + if (algorithm == COMPRESSION_ALGORITHM_LZMA || algorithm == COMPRESSION_ALGORITHM_IMLZMA) { + model->addInfo(index, usprintf("\nLZMA dictionary size: %Xh", dictionarySize)); + } // Update parsing data COMPRESSED_SECTION_PARSING_DATA pdata; pdata.algorithm = algorithm; + pdata.dictionarySize = dictionarySize; pdata.compressionType = compressionType; pdata.uncompressedSize = uncompressedSize; model->setParsingData(index, UByteArray((const char*)&pdata, sizeof(pdata))); @@ -2532,10 +2531,11 @@ USTATUS FfsParser::parseGuidedSectionBody(const UModelIndex & index) UString info; bool parseCurrentSection = true; UINT8 algorithm = COMPRESSION_ALGORITHM_NONE; + UINT32 dictionarySize = 0; UByteArray baGuid = UByteArray((const char*)&guid, sizeof(EFI_GUID)); // Tiano compressed section if (baGuid == EFI_GUIDED_SECTION_TIANO) { - USTATUS result = decompress(model->body(index), EFI_STANDARD_COMPRESSION, algorithm, processed, efiDecompressed); + USTATUS result = decompress(model->body(index), EFI_STANDARD_COMPRESSION, algorithm, dictionarySize, processed, efiDecompressed); if (result) { msg(usprintf("%s: decompression failed with error ", __FUNCTION__) + errorCodeToUString(result), index); return U_SUCCESS; @@ -2563,7 +2563,7 @@ USTATUS FfsParser::parseGuidedSectionBody(const UModelIndex & index) } // LZMA compressed section else if (baGuid == EFI_GUIDED_SECTION_LZMA || baGuid == EFI_GUIDED_SECTION_LZMAF86) { - USTATUS result = decompress(model->body(index), EFI_CUSTOMIZED_COMPRESSION, algorithm, processed, efiDecompressed); + USTATUS result = decompress(model->body(index), EFI_CUSTOMIZED_COMPRESSION, algorithm, dictionarySize, processed, efiDecompressed); if (result) { msg(usprintf("%s: decompression failed with error ", __FUNCTION__) + errorCodeToUString(result), index); return U_SUCCESS; @@ -2572,6 +2572,7 @@ USTATUS FfsParser::parseGuidedSectionBody(const UModelIndex & index) if (algorithm == COMPRESSION_ALGORITHM_LZMA) { info += UString("\nCompression algorithm: LZMA"); info += usprintf("\nDecompressed size: %Xh (%u)", processed.size(), processed.size()); + info += usprintf("\nLZMA dictionary size: %Xh", dictionarySize); } else { info += UString("\nCompression algorithm: unknown"); @@ -2597,6 +2598,11 @@ USTATUS FfsParser::parseGuidedSectionBody(const UModelIndex & index) if (algorithm != COMPRESSION_ALGORITHM_NONE) model->setCompressed(index, true); + // Set parsing data + GUIDED_SECTION_PARSING_DATA pdata; + pdata.dictionarySize = dictionarySize; + model->setParsingData(index, UByteArray((const char*)&pdata, sizeof(pdata))); + if (!parseCurrentSection) { msg(usprintf("%s: GUID defined section can not be processed", __FUNCTION__), index); return U_SUCCESS; diff --git a/common/parsingdata.h b/common/parsingdata.h index 682770d..ad3ca54 100644 --- a/common/parsingdata.h +++ b/common/parsingdata.h @@ -37,14 +37,20 @@ typedef struct FILE_PARSING_DATA_ { EFI_GUID guid; } FILE_PARSING_DATA; -typedef struct GUID_PARSING_DATA_ { +typedef struct GUIDED_SECTION_PARSING_DATA_ { EFI_GUID guid; -} GUIDED_SECTION_PARSING_DATA, FREEFORM_GUIDED_SECTION_PARSING_DATA; + UINT32 dictionarySize; +} GUIDED_SECTION_PARSING_DATA; + +typedef struct FREEFORM_GUIDED_SECTION_PARSING_DATA_ { + EFI_GUID guid; +} FREEFORM_GUIDED_SECTION_PARSING_DATA; typedef struct COMPRESSED_SECTION_PARSING_DATA_ { UINT32 uncompressedSize; UINT8 compressionType; UINT8 algorithm; + UINT32 dictionarySize; } COMPRESSED_SECTION_PARSING_DATA; typedef struct TE_IMAGE_SECTION_PARSING_DATA_ { diff --git a/common/treemodel.cpp b/common/treemodel.cpp index aa34478..0f5d3d5 100644 --- a/common/treemodel.cpp +++ b/common/treemodel.cpp @@ -296,11 +296,13 @@ void TreeModel::setFixed(const UModelIndex &index, const bool fixed) return; if (fixed) { + // Special handling for uncompressed to compressed boundary if (item->compressed() && item->parent()->compressed() == FALSE) { item->setFixed(item->parent()->fixed()); return; } + // Propagate fixed flag until root if (item->parent()->type() != Types::Root) item->parent()->setFixed(fixed); } diff --git a/common/utility.cpp b/common/utility.cpp index 3658f4b..a3b2f98 100644 --- a/common/utility.cpp +++ b/common/utility.cpp @@ -136,7 +136,7 @@ UString errorCodeToUString(USTATUS errorCode) } // Compression routines -USTATUS decompress(const UByteArray & compressedData, const UINT8 compressionType, UINT8 & algorithm, UByteArray & decompressedData, UByteArray & efiDecompressedData) +USTATUS decompress(const UByteArray & compressedData, const UINT8 compressionType, UINT8 & algorithm, UINT32 & dictionarySize, UByteArray & decompressedData, UByteArray & efiDecompressedData) { const UINT8* data; UINT32 dataSize; @@ -147,6 +147,9 @@ USTATUS decompress(const UByteArray & compressedData, const UINT8 compressionTyp UINT32 scratchSize = 0; const EFI_TIANO_HEADER* header; + // For all but LZMA dictionary size is 0 + dictionarySize = 0; + switch (compressionType) { case EFI_NOT_COMPRESSED: @@ -189,13 +192,9 @@ USTATUS decompress(const UByteArray & compressedData, const UINT8 compressionTyp USTATUS EfiResult = EfiDecompress(data, dataSize, efiDecompressed, decompressedSize, scratch, scratchSize); if (decompressedSize > INT32_MAX) { - free(decompressed); - free(efiDecompressed); - free(scratch); - return U_STANDARD_DECOMPRESSION_FAILED; + result = U_STANDARD_DECOMPRESSION_FAILED; } - - if (EfiResult == U_SUCCESS && TianoResult == U_SUCCESS) { // Both decompressions are OK + else if (EfiResult == U_SUCCESS && TianoResult == U_SUCCESS) { // Both decompressions are OK algorithm = COMPRESSION_ALGORITHM_UNDECIDED; decompressedData = UByteArray((const char*)decompressed, (int)decompressedSize); efiDecompressedData = UByteArray((const char*)efiDecompressed, (int)decompressedSize); @@ -239,6 +238,9 @@ USTATUS decompress(const UByteArray & compressedData, const UINT8 compressionTyp if (U_SUCCESS != LzmaDecompress(data, dataSize, decompressed)) { // Intel modified LZMA workaround // Decompress section data once again + + // VERIFY: might be wrong assumption, 0.2x had a different code here + // See: https://github.com/LongSoft/UEFITool/blob/4bee991c949b458739ffa96b88dbc589192c7689/ffsengine.cpp#L2814-L2823 data += sizeof(UINT32); // Get info again @@ -258,6 +260,7 @@ USTATUS decompress(const UByteArray & compressedData, const UINT8 compressionTyp return U_CUSTOMIZED_DECOMPRESSION_FAILED; } algorithm = COMPRESSION_ALGORITHM_IMLZMA; + dictionarySize = readUnaligned((UINT32*)(data + 1)); // LZMA dictionary size is stored in bytes 1-4 of LZMA properties header decompressedData = UByteArray((const char*)decompressed, (int)decompressedSize); } } @@ -267,6 +270,7 @@ USTATUS decompress(const UByteArray & compressedData, const UINT8 compressionTyp return U_CUSTOMIZED_DECOMPRESSION_FAILED; } algorithm = COMPRESSION_ALGORITHM_LZMA; + dictionarySize = readUnaligned((UINT32*)(data + 1)); // LZMA dictionary size is stored in bytes 1-4 of LZMA properties header decompressedData = UByteArray((const char*)decompressed, (int)decompressedSize); } @@ -414,7 +418,7 @@ USTATUS gzipDecompress(const UByteArray & input, UByteArray & output) // 15 for the maximum history buffer, 16 for gzip only input. int ret = inflateInit2(&stream, 15U | 16U); if (ret != Z_OK) - return U_CUSTOMIZED_DECOMPRESSION_FAILED; + return U_GZIP_DECOMPRESSION_FAILED; while (ret == Z_OK) { Bytef out[4096]; @@ -427,5 +431,5 @@ USTATUS gzipDecompress(const UByteArray & input, UByteArray & output) } inflateEnd(&stream); - return ret == Z_STREAM_END ? U_SUCCESS : U_CUSTOMIZED_DECOMPRESSION_FAILED; + return ret == Z_STREAM_END ? U_SUCCESS : U_GZIP_DECOMPRESSION_FAILED; } diff --git a/common/utility.h b/common/utility.h index addc48d..7bad658 100644 --- a/common/utility.h +++ b/common/utility.h @@ -29,8 +29,8 @@ UString uniqueItemName(const UModelIndex & index); // Converts error code to UString UString errorCodeToUString(USTATUS errorCode); -// EFI/Tiano decompression routine -USTATUS decompress(const UByteArray & compressed, const UINT8 compressionType, UINT8 & algorithm, UByteArray & decompressed, UByteArray & efiDecompressed); +// EFI/Tiano/LZMA decompression routine +USTATUS decompress(const UByteArray & compressed, const UINT8 compressionType, UINT8 & algorithm, UINT32 & dictionarySize, UByteArray & decompressed, UByteArray & efiDecompressed); // GZIP decompression routine USTATUS gzipDecompress(const UByteArray & compressed, UByteArray & decompressed);