From 934ce1f3f8b320f0adb04f960a3031fd2b37d602 Mon Sep 17 00:00:00 2001 From: Nikolaj Schlej Date: Mon, 29 Aug 2022 08:23:38 +0200 Subject: [PATCH] Kaitai-based Intel ACM and BootGuard parsers As the first step towards automated parsing, this change set replaces outdated BootGuard-related parsers with shiny new KaitaiStruct-based ones. It also does the following: - improves Intel FIT definitions by using the relevant specification - adds sha1, sha384, sha512 and sm3 digest implementations - updates LZMA SDK to v22.01 - moves GUIDs out of include files to prevent multiple instantiations - enforces C++11 - adds Kaitai-based parsers for Intel FIT, BootGuard v1 and BootGuard v2 structures - makes many small refactorings here, there and everywhere --- UEFIExtract/CMakeLists.txt | 63 +- UEFIFind/CMakeLists.txt | 63 +- UEFITool/CMakeLists.txt | 167 +- UEFITool/ffsfinder.cpp | 2 +- UEFITool/uefitool.pro | 44 +- common/LZMA/LzmaCompress.c | 6 +- common/LZMA/LzmaCompress.h | 2 +- common/LZMA/LzmaDecompress.c | 6 +- common/LZMA/SDK/C/7zTypes.h | 529 +++ common/LZMA/SDK/C/7zVersion.h | 32 +- common/LZMA/SDK/C/Bra.c | 230 ++ common/LZMA/SDK/C/Bra.h | 12 +- common/LZMA/SDK/C/Bra86.c | 99 +- common/LZMA/SDK/C/Compiler.h | 43 + common/LZMA/SDK/C/CpuArch.c | 478 +++ common/LZMA/SDK/C/CpuArch.h | 404 ++- common/LZMA/SDK/C/LzFind.c | 1968 ++++++++--- common/LZMA/SDK/C/LzFind.h | 83 +- common/LZMA/SDK/C/LzHash.h | 60 +- common/LZMA/SDK/C/LzmaDec.c | 1828 ++++++---- common/LZMA/SDK/C/LzmaDec.h | 59 +- common/LZMA/SDK/C/LzmaEnc.c | 4390 +++++++++++++++---------- common/LZMA/SDK/C/LzmaEnc.h | 58 +- common/LZMA/SDK/C/Precomp.h | 12 + common/LZMA/SDK/C/Types.h | 256 -- common/basetypes.h | 29 +- common/bootguard.h | 210 -- common/bstrlib/bstrlib.h | 2 +- common/bstrlib/bstrwrap.h | 3 +- common/descriptor.cpp | 2 +- common/digest/sha1.c | 230 ++ common/{uinttypes.h => digest/sha1.h} | 28 +- common/{sha256.h => digest/sha2.h} | 16 +- common/digest/sha256.c | 279 ++ common/digest/sha512.c | 312 ++ common/digest/sm3.c | 234 ++ common/digest/sm3.h | 32 + common/ffs.cpp | 112 +- common/ffs.h | 153 +- common/ffsparser.cpp | 1072 ++---- common/ffsparser.h | 63 +- common/ffsreport.cpp | 1 - common/fitparser.cpp | 1116 +++++++ common/fitparser.h | 100 + common/generated/intel_acbp_v1.cpp | 472 +++ common/generated/intel_acbp_v1.h | 457 +++ common/generated/intel_acbp_v2.cpp | 443 +++ common/generated/intel_acbp_v2.h | 441 +++ common/generated/intel_acm.cpp | 92 + common/generated/intel_acm.h | 159 + common/generated/intel_keym_v1.cpp | 162 + common/generated/intel_keym_v1.h | 177 + common/generated/intel_keym_v2.cpp | 214 ++ common/generated/intel_keym_v2.h | 224 ++ common/intel_fit.h | 226 ++ common/{fit.h => intel_microcode.h} | 53 +- common/kaitai/custom_decoder.h | 16 + common/kaitai/exceptions.h | 189 ++ common/kaitai/kaitaistream.cpp | 693 ++++ common/kaitai/kaitaistream.h | 295 ++ common/kaitai/kaitaistruct.h | 20 + common/ksy/intel_acbp_v1.ksy | 202 ++ common/ksy/intel_acbp_v2.ksy | 223 ++ common/ksy/intel_acm.ksy | 94 + common/ksy/intel_keym_v1.ksy | 79 + common/ksy/intel_keym_v2.ksy | 109 + common/me.h | 6 +- common/meparser.cpp | 9 +- common/meparser.h | 3 +- common/meson.build | 12 +- common/nvram.cpp | 60 + common/nvram.h | 130 +- common/nvramparser.cpp | 17 +- common/peimage.cpp | 2 +- common/sha256.c | 212 -- common/treemodel.cpp | 2 +- common/types.cpp | 64 +- common/types.h | 1 + common/ustring.h | 3 +- common/utility.cpp | 1 + kaitai_regenerate.sh | 41 + 81 files changed, 15212 insertions(+), 5279 deletions(-) create mode 100644 common/LZMA/SDK/C/7zTypes.h create mode 100644 common/LZMA/SDK/C/Bra.c create mode 100644 common/LZMA/SDK/C/Compiler.h create mode 100644 common/LZMA/SDK/C/CpuArch.c create mode 100644 common/LZMA/SDK/C/Precomp.h delete mode 100644 common/LZMA/SDK/C/Types.h delete mode 100644 common/bootguard.h create mode 100644 common/digest/sha1.c rename common/{uinttypes.h => digest/sha1.h} (50%) rename common/{sha256.h => digest/sha2.h} (69%) create mode 100644 common/digest/sha256.c create mode 100644 common/digest/sha512.c create mode 100644 common/digest/sm3.c create mode 100644 common/digest/sm3.h create mode 100644 common/fitparser.cpp create mode 100644 common/fitparser.h create mode 100644 common/generated/intel_acbp_v1.cpp create mode 100644 common/generated/intel_acbp_v1.h create mode 100644 common/generated/intel_acbp_v2.cpp create mode 100644 common/generated/intel_acbp_v2.h create mode 100644 common/generated/intel_acm.cpp create mode 100644 common/generated/intel_acm.h create mode 100644 common/generated/intel_keym_v1.cpp create mode 100644 common/generated/intel_keym_v1.h create mode 100644 common/generated/intel_keym_v2.cpp create mode 100644 common/generated/intel_keym_v2.h create mode 100755 common/intel_fit.h rename common/{fit.h => intel_microcode.h} (72%) mode change 100755 => 100644 create mode 100644 common/kaitai/custom_decoder.h create mode 100644 common/kaitai/exceptions.h create mode 100644 common/kaitai/kaitaistream.cpp create mode 100644 common/kaitai/kaitaistream.h create mode 100644 common/kaitai/kaitaistruct.h create mode 100644 common/ksy/intel_acbp_v1.ksy create mode 100644 common/ksy/intel_acbp_v2.ksy create mode 100644 common/ksy/intel_acm.ksy create mode 100644 common/ksy/intel_keym_v1.ksy create mode 100644 common/ksy/intel_keym_v2.ksy delete mode 100644 common/sha256.c create mode 100755 kaitai_regenerate.sh diff --git a/UEFIExtract/CMakeLists.txt b/UEFIExtract/CMakeLists.txt index 0050f2b..975e109 100644 --- a/UEFIExtract/CMakeLists.txt +++ b/UEFIExtract/CMakeLists.txt @@ -1,7 +1,11 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.0) +CMAKE_MINIMUM_REQUIRED(VERSION 3.1.0 FATAL_ERROR) PROJECT(UEFIExtract) +SET(CMAKE_CXX_STANDARD 11) +SET(CMAKE_CXX_STANDARD_REQUIRED ON) +SET(CMAKE_CXX_EXTENSIONS OFF) + SET(PROJECT_SOURCES uefiextract_main.cpp ffsdumper.cpp @@ -14,19 +18,31 @@ SET(PROJECT_SOURCES ../common/nvramparser.cpp ../common/meparser.cpp ../common/ffsparser.cpp + ../common/fitparser.cpp ../common/ffsreport.cpp ../common/peimage.cpp ../common/treeitem.cpp ../common/treemodel.cpp ../common/utility.cpp ../common/LZMA/LzmaDecompress.c + ../common/LZMA/SDK/C/Bra.c ../common/LZMA/SDK/C/Bra86.c + ../common/LZMA/SDK/C/CpuArch.c ../common/LZMA/SDK/C/LzmaDec.c ../common/Tiano/EfiTianoDecompress.c ../common/ustring.cpp - ../common/sha256.c ../common/bstrlib/bstrlib.c ../common/bstrlib/bstrwrap.cpp + ../common/generated/intel_acbp_v1.cpp + ../common/generated/intel_acbp_v2.cpp + ../common/generated/intel_keym_v1.cpp + ../common/generated/intel_keym_v2.cpp + ../common/generated/intel_acm.cpp + ../common/kaitai/kaitaistream.cpp + ../common/digest/sha1.c + ../common/digest/sha256.c + ../common/digest/sha512.c + ../common/digest/sm3.c ../common/zlib/adler32.c ../common/zlib/compress.c ../common/zlib/crc32.c @@ -44,48 +60,7 @@ SET(PROJECT_SOURCES ../common/zlib/zutil.c ) -SET(PROJECT_HEADERS - ffsdumper.h - uefidump.h - ../common/guiddatabase.h - ../common/basetypes.h - ../common/descriptor.h - ../common/gbe.h - ../common/me.h - ../common/ffs.h - ../common/nvram.h - ../common/nvramparser.h - ../common/ffsparser.h - ../common/ffsreport.h - ../common/peimage.h - ../common/types.h - ../common/treeitem.h - ../common/treemodel.h - ../common/utility.h - ../common/LZMA/LzmaDecompress.h - ../common/Tiano/EfiTianoDecompress.h - ../common/ubytearray.h - ../common/ustring.h - ../common/bootguard.h - ../common/sha256.h - ../common/filesystem.h - ../common/bstrlib/bstrlib.h - ../common/bstrlib/bstrwrap.h - ../common/zlib/zconf.h - ../common/zlib/zlib.h - ../common/zlib/crc32.h - ../common/zlib/deflate.h - ../common/zlib/gzguts.h - ../common/zlib/inffast.h - ../common/zlib/inffixed.h - ../common/zlib/inflate.h - ../common/zlib/inftrees.h - ../common/zlib/trees.h - ../common/zlib/zutil.h - ../version.h -) - ADD_DEFINITIONS(-DU_ENABLE_NVRAM_PARSING_SUPPORT -DU_ENABLE_ME_PARSING_SUPPORT -DU_ENABLE_FIT_PARSING_SUPPORT -DU_ENABLE_GUID_DATABASE_SUPPORT) -ADD_EXECUTABLE(UEFIExtract ${PROJECT_SOURCES} ${PROJECT_HEADERS}) +ADD_EXECUTABLE(UEFIExtract ${PROJECT_SOURCES}) diff --git a/UEFIFind/CMakeLists.txt b/UEFIFind/CMakeLists.txt index 1d31949..09951c7 100644 --- a/UEFIFind/CMakeLists.txt +++ b/UEFIFind/CMakeLists.txt @@ -1,7 +1,11 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.0) +CMAKE_MINIMUM_REQUIRED(VERSION 3.1.0 FATAL_ERROR) PROJECT(UEFIFind) +SET(CMAKE_CXX_STANDARD 11) +SET(CMAKE_CXX_STANDARD_REQUIRED ON) +SET(CMAKE_CXX_EXTENSIONS OFF) + SET(PROJECT_SOURCES uefifind_main.cpp uefifind.cpp @@ -12,6 +16,7 @@ SET(PROJECT_SOURCES ../common/nvram.cpp ../common/nvramparser.cpp ../common/ffsparser.cpp + ../common/fitparser.cpp ../common/ffsreport.cpp ../common/ffsutils.cpp ../common/peimage.cpp @@ -19,13 +24,24 @@ SET(PROJECT_SOURCES ../common/treemodel.cpp ../common/utility.cpp ../common/LZMA/LzmaDecompress.c + ../common/LZMA/SDK/C/Bra.c ../common/LZMA/SDK/C/Bra86.c + ../common/LZMA/SDK/C/CpuArch.c ../common/LZMA/SDK/C/LzmaDec.c ../common/Tiano/EfiTianoDecompress.c ../common/ustring.cpp - ../common/sha256.c ../common/bstrlib/bstrlib.c ../common/bstrlib/bstrwrap.cpp + ../common/generated/intel_acbp_v1.cpp + ../common/generated/intel_acbp_v2.cpp + ../common/generated/intel_keym_v1.cpp + ../common/generated/intel_keym_v2.cpp + ../common/generated/intel_acm.cpp + ../common/kaitai/kaitaistream.cpp + ../common/digest/sha1.c + ../common/digest/sha256.c + ../common/digest/sha512.c + ../common/digest/sm3.c ../common/zlib/adler32.c ../common/zlib/compress.c ../common/zlib/crc32.c @@ -43,48 +59,7 @@ SET(PROJECT_SOURCES ../common/zlib/zutil.c ) -SET(PROJECT_HEADERS - uefifind.h - ../common/guiddatabase.h - ../common/basetypes.h - ../common/descriptor.h - ../common/gbe.h - ../common/me.h - ../common/ffs.h - ../common/nvram.h - ../common/nvramparser.h - ../common/ffsparser.h - ../common/ffsreport.h - ../common/ffsutils.h - ../common/peimage.h - ../common/types.h - ../common/treeitem.h - ../common/treemodel.h - ../common/utility.h - ../common/LZMA/LzmaDecompress.h - ../common/Tiano/EfiTianoDecompress.h - ../common/ubytearray.h - ../common/ustring.h - ../common/bootguard.h - ../common/sha256.h - ../common/filesystem.h - ../common/bstrlib/bstrlib.h - ../common/bstrlib/bstrwrap.h - ../common/zlib/zconf.h - ../common/zlib/zlib.h - ../common/zlib/crc32.h - ../common/zlib/deflate.h - ../common/zlib/gzguts.h - ../common/zlib/inffast.h - ../common/zlib/inffixed.h - ../common/zlib/inflate.h - ../common/zlib/inftrees.h - ../common/zlib/trees.h - ../common/zlib/zutil.h - ../version.h -) - ADD_DEFINITIONS(-DU_ENABLE_NVRAM_PARSING_SUPPORT -DU_ENABLE_FIT_PARSING_SUPPORT -DU_ENABLE_GUID_DATABASE_SUPPORT) -ADD_EXECUTABLE(UEFIFind ${PROJECT_SOURCES} ${PROJECT_HEADERS}) +ADD_EXECUTABLE(UEFIFind ${PROJECT_SOURCES}) diff --git a/UEFITool/CMakeLists.txt b/UEFITool/CMakeLists.txt index cfab613..f1d942f 100644 --- a/UEFITool/CMakeLists.txt +++ b/UEFITool/CMakeLists.txt @@ -2,6 +2,10 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.16) PROJECT(UEFITool LANGUAGES C CXX) +SET(CMAKE_CXX_STANDARD 11) +SET(CMAKE_CXX_STANDARD_REQUIRED ON) +SET(CMAKE_CXX_EXTENSIONS OFF) + FIND_PACKAGE(Qt6 REQUIRED COMPONENTS Widgets) QT_STANDARD_PROJECT_SETUP() @@ -13,6 +17,15 @@ SET(PROJECT_FORMS gotoaddressdialog.ui ) +SET(PROJECT_HEADERS + uefitool.h + hexspinbox.h + searchdialog.h + hexviewdialog.h + gotobasedialog.h + gotoaddressdialog.h +) + SET(PROJECT_SOURCES icons/uefitool.icns uefitool.rc @@ -23,108 +36,64 @@ SET(PROJECT_SOURCES guidlineedit.cpp ffsfinder.cpp hexspinbox.cpp - ../common/guiddatabase.cpp - ../common/nvram.cpp - ../common/nvramparser.cpp - ../common/meparser.cpp - ../common/ffsops.cpp - ../common/types.cpp - ../common/descriptor.cpp - ../common/ffs.cpp - ../common/peimage.cpp - ../common/utility.cpp - ../common/ffsbuilder.cpp - ../common/ffsparser.cpp - ../common/ffsreport.cpp - ../common/ffsutils.cpp - ../common/treeitem.cpp - ../common/treemodel.cpp - ../common/LZMA/LzmaCompress.c - ../common/LZMA/LzmaDecompress.c - ../common/LZMA/SDK/C/Bra86.c - ../common/LZMA/SDK/C/LzFind.c - ../common/LZMA/SDK/C/LzmaDec.c - ../common/LZMA/SDK/C/LzmaEnc.c - ../common/Tiano/EfiTianoDecompress.c - ../common/Tiano/EfiTianoCompress.c - ../common/Tiano/EfiTianoCompressLegacy.c - ../common/ustring.cpp - ../common/sha256.c - ../common/zlib/adler32.c - ../common/zlib/compress.c - ../common/zlib/crc32.c - ../common/zlib/deflate.c - ../common/zlib/gzclose.c - ../common/zlib/gzlib.c - ../common/zlib/gzread.c - ../common/zlib/gzwrite.c - ../common/zlib/inflate.c - ../common/zlib/infback.c - ../common/zlib/inftrees.c - ../common/zlib/inffast.c - ../common/zlib/trees.c - ../common/zlib/uncompr.c - ../common/zlib/zutil.c qhexedit2/qhexedit.cpp qhexedit2/chunks.cpp qhexedit2/commands.cpp - ) - -SET(PROJECT_HEADERS - hexspinbox.h - uefitool.h - searchdialog.h - hexviewdialog.h - gotobasedialog.h - gotoaddressdialog.h - guidlineedit.h - ffsfinder.h - ../common/guiddatabase.h - ../common/nvram.h - ../common/nvramparser.h - ../common/meparser.h - ../common/ffsops.h - ../common/basetypes.h - ../common/descriptor.h - ../common/gbe.h - ../common/me.h - ../common/ffs.h - ../common/fit.h - ../common/peimage.h - ../common/types.h - ../common/utility.h - ../common/parsingdata.h - ../common/ffsbuilder.h - ../common/ffsparser.h - ../common/ffsreport.h - ../common/treeitem.h - ../common/ffsutils.h - ../common/treemodel.h - ../common/LZMA/LzmaCompress.h - ../common/LZMA/LzmaDecompress.h - ../common/Tiano/EfiTianoDecompress.h - ../common/Tiano/EfiTianoCompress.h - ../common/uinttypes.h - ../common/ustring.h - ../common/ubytearray.h - ../common/bootguard.h - ../common/sha256.h - ../common/zlib/zconf.h - ../common/zlib/zlib.h - ../common/zlib/crc32.h - ../common/zlib/deflate.h - ../common/zlib/gzguts.h - ../common/zlib/inffast.h - ../common/zlib/inffixed.h - ../common/zlib/inflate.h - ../common/zlib/inftrees.h - ../common/zlib/trees.h - ../common/zlib/zutil.h - ../version.h - qhexedit2/qhexedit.h - qhexedit2/chunks.h - qhexedit2/commands.h - ) + ../common/fitparser.cpp + ../common/guiddatabase.cpp + ../common/nvram.cpp + ../common/nvramparser.cpp + ../common/meparser.cpp + ../common/ffsops.cpp + ../common/types.cpp + ../common/descriptor.cpp + ../common/ffs.cpp + ../common/peimage.cpp + ../common/utility.cpp + ../common/ffsbuilder.cpp + ../common/ffsparser.cpp + ../common/ffsreport.cpp + ../common/ffsutils.cpp + ../common/treeitem.cpp + ../common/treemodel.cpp + ../common/LZMA/LzmaCompress.c + ../common/LZMA/LzmaDecompress.c + ../common/LZMA/SDK/C/CpuArch.c + ../common/LZMA/SDK/C/Bra.c + ../common/LZMA/SDK/C/Bra86.c + ../common/LZMA/SDK/C/LzFind.c + ../common/LZMA/SDK/C/LzmaDec.c + ../common/LZMA/SDK/C/LzmaEnc.c + ../common/Tiano/EfiTianoDecompress.c + ../common/Tiano/EfiTianoCompress.c + ../common/Tiano/EfiTianoCompressLegacy.c + ../common/ustring.cpp + ../common/digest/sha1.c + ../common/digest/sha256.c + ../common/digest/sha512.c + ../common/digest/sm3.c + ../common/generated/intel_acbp_v1.cpp + ../common/generated/intel_acbp_v2.cpp + ../common/generated/intel_keym_v1.cpp + ../common/generated/intel_keym_v2.cpp + ../common/generated/intel_acm.cpp + ../common/kaitai/kaitaistream.cpp + ../common/zlib/adler32.c + ../common/zlib/compress.c + ../common/zlib/crc32.c + ../common/zlib/deflate.c + ../common/zlib/gzclose.c + ../common/zlib/gzlib.c + ../common/zlib/gzread.c + ../common/zlib/gzwrite.c + ../common/zlib/inflate.c + ../common/zlib/infback.c + ../common/zlib/inftrees.c + ../common/zlib/inffast.c + ../common/zlib/trees.c + ../common/zlib/uncompr.c + ../common/zlib/zutil.c +) QT_ADD_RESOURCES(PROJECT_SOURCES uefitool.qrc diff --git a/UEFITool/ffsfinder.cpp b/UEFITool/ffsfinder.cpp index 2a632b5..fd98ce5 100644 --- a/UEFITool/ffsfinder.cpp +++ b/UEFITool/ffsfinder.cpp @@ -21,7 +21,7 @@ USTATUS FfsFinder::findHexPattern(const UModelIndex & index, const UByteArray & hexPattern, const UINT8 mode) { - //TODO: use FfsUtils. + // TODO: use FfsUtils. if (!index.isValid()) return U_SUCCESS; diff --git a/UEFITool/uefitool.pro b/UEFITool/uefitool.pro index 728b416..023c563 100644 --- a/UEFITool/uefitool.pro +++ b/UEFITool/uefitool.pro @@ -2,6 +2,9 @@ QT += core gui widgets TARGET = UEFITool TEMPLATE = app + +CONFIG += c++11 + DEFINES += "U_ENABLE_FIT_PARSING_SUPPORT" DEFINES += "U_ENABLE_NVRAM_PARSING_SUPPORT" DEFINES += "U_ENABLE_ME_PARSING_SUPPORT" @@ -15,6 +18,7 @@ HEADERS += uefitool.h \ guidlineedit.h \ ffsfinder.h \ hexspinbox.h \ + ../common/fitparser.h \ ../common/guiddatabase.h \ ../common/nvram.h \ ../common/nvramparser.h \ @@ -25,7 +29,6 @@ HEADERS += uefitool.h \ ../common/gbe.h \ ../common/me.h \ ../common/ffs.h \ - ../common/fit.h \ ../common/peimage.h \ ../common/types.h \ ../common/utility.h \ @@ -35,27 +38,28 @@ HEADERS += uefitool.h \ ../common/ffsreport.h \ ../common/treeitem.h \ ../common/ffsutils.h \ + ../common/intel_fit.h \ + ../common/intel_microcode.h \ ../common/treemodel.h \ ../common/LZMA/LzmaCompress.h \ ../common/LZMA/LzmaDecompress.h \ ../common/Tiano/EfiTianoDecompress.h \ ../common/Tiano/EfiTianoCompress.h \ - ../common/uinttypes.h \ ../common/ustring.h \ ../common/ubytearray.h \ - ../common/bootguard.h \ - ../common/sha256.h \ - ../common/zlib/zconf.h \ + ../common/digest/sha1.h \ + ../common/digest/sha2.h \ + ../common/digest/sm3.h \ + ../common/generated/intel_acbp_v1.h \ + ../common/generated/intel_acbp_v2.h \ + ../common/generated/intel_keym_v1.h \ + ../common/generated/intel_keym_v2.h \ + ../common/generated/intel_acm.h \ + ../common/kaitai/kaitaistream.h \ + ../common/kaitai/kaitaistruct.h \ + ../common/kaitai/exceptions.h \ ../common/zlib/zlib.h \ ../common/zlib/crc32.h \ - ../common/zlib/deflate.h \ - ../common/zlib/gzguts.h \ - ../common/zlib/inffast.h \ - ../common/zlib/inffixed.h \ - ../common/zlib/inflate.h \ - ../common/zlib/inftrees.h \ - ../common/zlib/trees.h \ - ../common/zlib/zutil.h \ ../version.h \ qhexedit2/qhexedit.h \ qhexedit2/chunks.h \ @@ -68,6 +72,7 @@ SOURCES += uefitool_main.cpp \ guidlineedit.cpp \ ffsfinder.cpp \ hexspinbox.cpp \ + ../common/fitparser.cpp \ ../common/guiddatabase.cpp \ ../common/nvram.cpp \ ../common/nvramparser.cpp \ @@ -86,6 +91,8 @@ SOURCES += uefitool_main.cpp \ ../common/treemodel.cpp \ ../common/LZMA/LzmaCompress.c \ ../common/LZMA/LzmaDecompress.c \ + ../common/LZMA/SDK/C/CpuArch.c \ + ../common/LZMA/SDK/C/Bra.c \ ../common/LZMA/SDK/C/Bra86.c \ ../common/LZMA/SDK/C/LzFind.c \ ../common/LZMA/SDK/C/LzmaDec.c \ @@ -94,7 +101,16 @@ SOURCES += uefitool_main.cpp \ ../common/Tiano/EfiTianoCompress.c \ ../common/Tiano/EfiTianoCompressLegacy.c \ ../common/ustring.cpp \ - ../common/sha256.c \ + ../common/digest/sha1.c \ + ../common/digest/sha256.c \ + ../common/digest/sha512.c \ + ../common/digest/sm3.c \ + ../common/generated/intel_acbp_v1.cpp \ + ../common/generated/intel_acbp_v2.cpp \ + ../common/generated/intel_keym_v1.cpp \ + ../common/generated/intel_keym_v2.cpp \ + ../common/generated/intel_acm.cpp \ + ../common/kaitai/kaitaistream.cpp \ ../common/zlib/adler32.c \ ../common/zlib/compress.c \ ../common/zlib/crc32.c \ diff --git a/common/LZMA/LzmaCompress.c b/common/LZMA/LzmaCompress.c index 25b086b..dc97fa5 100644 --- a/common/LZMA/LzmaCompress.c +++ b/common/LZMA/LzmaCompress.c @@ -19,11 +19,11 @@ WITHWARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. #define LZMA_HEADER_SIZE (LZMA_PROPS_SIZE + 8) -static void * AllocForLzma(void *p, size_t size) { (void)p; return malloc(size); } -static void FreeForLzma(void *p, void *address) { (void)p; free(address); } +static void * AllocForLzma(ISzAllocPtr p, size_t size) { (void)p; return malloc(size); } +static void FreeForLzma(ISzAllocPtr p, void *address) { (void)p; free(address); } static ISzAlloc SzAllocForLzma = { &AllocForLzma, &FreeForLzma }; -SRes OnProgress(void *p, UInt64 inSize, UInt64 outSize) +SRes OnProgress(const ICompressProgress *p, UInt64 inSize, UInt64 outSize) { (void)p; (void)inSize; (void)outSize; return SZ_OK; diff --git a/common/LZMA/LzmaCompress.h b/common/LZMA/LzmaCompress.h index 08bbe74..6036523 100644 --- a/common/LZMA/LzmaCompress.h +++ b/common/LZMA/LzmaCompress.h @@ -14,7 +14,7 @@ #ifndef LZMACOMPRESS_H #define LZMACOMPRESS_H -#include "SDK/C/Types.h" +#include "SDK/C/7zTypes.h" #include "../basetypes.h" #ifdef __cplusplus diff --git a/common/LZMA/LzmaDecompress.c b/common/LZMA/LzmaDecompress.c index 8ff8283..913ec97 100644 --- a/common/LZMA/LzmaDecompress.c +++ b/common/LZMA/LzmaDecompress.c @@ -12,7 +12,7 @@ WITHWARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. */ #include "LzmaDecompress.h" -#include "SDK/C/Types.h" +#include "SDK/C/7zTypes.h" #include "SDK/C/7zVersion.h" #include @@ -27,8 +27,8 @@ LShiftU64 ( return Operand << Count; } -static void * AllocForLzma(void *p, size_t size) { (void)p; return malloc(size); } -static void FreeForLzma(void *p, void *address) { (void)p; free(address); } +static void * AllocForLzma(ISzAllocPtr p, size_t size) { (void)p; return malloc(size); } +static void FreeForLzma(ISzAllocPtr p, void *address) { (void)p; free(address); } static ISzAlloc SzAllocForLzma = { &AllocForLzma, &FreeForLzma }; /* diff --git a/common/LZMA/SDK/C/7zTypes.h b/common/LZMA/SDK/C/7zTypes.h new file mode 100644 index 0000000..f7d7071 --- /dev/null +++ b/common/LZMA/SDK/C/7zTypes.h @@ -0,0 +1,529 @@ +/* 7zTypes.h -- Basic types +2022-04-01 : Igor Pavlov : Public domain */ + +#ifndef __7Z_TYPES_H +#define __7Z_TYPES_H + +#ifdef _WIN32 +/* #include */ +#else +#include +#endif + +#include + +#ifndef EXTERN_C_BEGIN +#ifdef __cplusplus +#define EXTERN_C_BEGIN extern "C" { +#define EXTERN_C_END } +#else +#define EXTERN_C_BEGIN +#define EXTERN_C_END +#endif +#endif + +EXTERN_C_BEGIN + +#define SZ_OK 0 + +#define SZ_ERROR_DATA 1 +#define SZ_ERROR_MEM 2 +#define SZ_ERROR_CRC 3 +#define SZ_ERROR_UNSUPPORTED 4 +#define SZ_ERROR_PARAM 5 +#define SZ_ERROR_INPUT_EOF 6 +#define SZ_ERROR_OUTPUT_EOF 7 +#define SZ_ERROR_READ 8 +#define SZ_ERROR_WRITE 9 +#define SZ_ERROR_PROGRESS 10 +#define SZ_ERROR_FAIL 11 +#define SZ_ERROR_THREAD 12 + +#define SZ_ERROR_ARCHIVE 16 +#define SZ_ERROR_NO_ARCHIVE 17 + +typedef int SRes; + + +#ifdef _MSC_VER + #if _MSC_VER > 1200 + #define MY_ALIGN(n) __declspec(align(n)) + #else + #define MY_ALIGN(n) + #endif +#else + #define MY_ALIGN(n) __attribute__ ((aligned(n))) +#endif + + +#ifdef _WIN32 + +/* typedef DWORD WRes; */ +typedef unsigned WRes; +#define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x) + +// #define MY_HRES_ERROR__INTERNAL_ERROR MY_SRes_HRESULT_FROM_WRes(ERROR_INTERNAL_ERROR) + +#else // _WIN32 + +// #define ENV_HAVE_LSTAT +typedef int WRes; + +// (FACILITY_ERRNO = 0x800) is 7zip's FACILITY constant to represent (errno) errors in HRESULT +#define MY__FACILITY_ERRNO 0x800 +#define MY__FACILITY_WIN32 7 +#define MY__FACILITY__WRes MY__FACILITY_ERRNO + +#define MY_HRESULT_FROM_errno_CONST_ERROR(x) ((HRESULT)( \ + ( (HRESULT)(x) & 0x0000FFFF) \ + | (MY__FACILITY__WRes << 16) \ + | (HRESULT)0x80000000 )) + +#define MY_SRes_HRESULT_FROM_WRes(x) \ + ((HRESULT)(x) <= 0 ? ((HRESULT)(x)) : MY_HRESULT_FROM_errno_CONST_ERROR(x)) + +// we call macro HRESULT_FROM_WIN32 for system errors (WRes) that are (errno) +#define HRESULT_FROM_WIN32(x) MY_SRes_HRESULT_FROM_WRes(x) + +/* +#define ERROR_FILE_NOT_FOUND 2L +#define ERROR_ACCESS_DENIED 5L +#define ERROR_NO_MORE_FILES 18L +#define ERROR_LOCK_VIOLATION 33L +#define ERROR_FILE_EXISTS 80L +#define ERROR_DISK_FULL 112L +#define ERROR_NEGATIVE_SEEK 131L +#define ERROR_ALREADY_EXISTS 183L +#define ERROR_DIRECTORY 267L +#define ERROR_TOO_MANY_POSTS 298L + +#define ERROR_INTERNAL_ERROR 1359L +#define ERROR_INVALID_REPARSE_DATA 4392L +#define ERROR_REPARSE_TAG_INVALID 4393L +#define ERROR_REPARSE_TAG_MISMATCH 4394L +*/ + +// we use errno equivalents for some WIN32 errors: + +#define ERROR_INVALID_PARAMETER EINVAL +#define ERROR_INVALID_FUNCTION EINVAL +#define ERROR_ALREADY_EXISTS EEXIST +#define ERROR_FILE_EXISTS EEXIST +#define ERROR_PATH_NOT_FOUND ENOENT +#define ERROR_FILE_NOT_FOUND ENOENT +#define ERROR_DISK_FULL ENOSPC +// #define ERROR_INVALID_HANDLE EBADF + +// we use FACILITY_WIN32 for errors that has no errno equivalent +// Too many posts were made to a semaphore. +#define ERROR_TOO_MANY_POSTS ((HRESULT)0x8007012AL) +#define ERROR_INVALID_REPARSE_DATA ((HRESULT)0x80071128L) +#define ERROR_REPARSE_TAG_INVALID ((HRESULT)0x80071129L) + +// if (MY__FACILITY__WRes != FACILITY_WIN32), +// we use FACILITY_WIN32 for COM errors: +#define E_OUTOFMEMORY ((HRESULT)0x8007000EL) +#define E_INVALIDARG ((HRESULT)0x80070057L) +#define MY__E_ERROR_NEGATIVE_SEEK ((HRESULT)0x80070083L) + +/* +// we can use FACILITY_ERRNO for some COM errors, that have errno equivalents: +#define E_OUTOFMEMORY MY_HRESULT_FROM_errno_CONST_ERROR(ENOMEM) +#define E_INVALIDARG MY_HRESULT_FROM_errno_CONST_ERROR(EINVAL) +#define MY__E_ERROR_NEGATIVE_SEEK MY_HRESULT_FROM_errno_CONST_ERROR(EINVAL) +*/ + +#define TEXT(quote) quote + +#define FILE_ATTRIBUTE_READONLY 0x0001 +#define FILE_ATTRIBUTE_HIDDEN 0x0002 +#define FILE_ATTRIBUTE_SYSTEM 0x0004 +#define FILE_ATTRIBUTE_DIRECTORY 0x0010 +#define FILE_ATTRIBUTE_ARCHIVE 0x0020 +#define FILE_ATTRIBUTE_DEVICE 0x0040 +#define FILE_ATTRIBUTE_NORMAL 0x0080 +#define FILE_ATTRIBUTE_TEMPORARY 0x0100 +#define FILE_ATTRIBUTE_SPARSE_FILE 0x0200 +#define FILE_ATTRIBUTE_REPARSE_POINT 0x0400 +#define FILE_ATTRIBUTE_COMPRESSED 0x0800 +#define FILE_ATTRIBUTE_OFFLINE 0x1000 +#define FILE_ATTRIBUTE_NOT_CONTENT_INDEXED 0x2000 +#define FILE_ATTRIBUTE_ENCRYPTED 0x4000 + +#define FILE_ATTRIBUTE_UNIX_EXTENSION 0x8000 /* trick for Unix */ + +#endif + + +#ifndef RINOK +#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; } +#endif + +#ifndef RINOK_WRes +#define RINOK_WRes(x) { WRes __result__ = (x); if (__result__ != 0) return __result__; } +#endif + +typedef unsigned char Byte; +typedef short Int16; +typedef unsigned short UInt16; + +#ifdef _LZMA_UINT32_IS_ULONG +typedef long Int32; +typedef unsigned long UInt32; +#else +typedef int Int32; +typedef unsigned int UInt32; +#endif + + +#ifndef _WIN32 + +typedef int INT; +typedef Int32 INT32; +typedef unsigned int UINT; +typedef UInt32 UINT32; +typedef INT32 LONG; // LONG, ULONG and DWORD must be 32-bit for _WIN32 compatibility +typedef UINT32 ULONG; + +#undef DWORD +typedef UINT32 DWORD; + +#define VOID void + +#define HRESULT LONG + +typedef void *LPVOID; +// typedef void VOID; +// typedef ULONG_PTR DWORD_PTR, *PDWORD_PTR; +// gcc / clang on Unix : sizeof(long==sizeof(void*) in 32 or 64 bits) +typedef long INT_PTR; +typedef unsigned long UINT_PTR; +typedef long LONG_PTR; +typedef unsigned long DWORD_PTR; + +typedef size_t SIZE_T; + +#endif // _WIN32 + + +#define MY_HRES_ERROR__INTERNAL_ERROR ((HRESULT)0x8007054FL) + + +#ifdef _SZ_NO_INT_64 + +/* define _SZ_NO_INT_64, if your compiler doesn't support 64-bit integers. + NOTES: Some code will work incorrectly in that case! */ + +typedef long Int64; +typedef unsigned long UInt64; + +#else + +#if defined(_MSC_VER) || defined(__BORLANDC__) +typedef __int64 Int64; +typedef unsigned __int64 UInt64; +#define UINT64_CONST(n) n +#else +typedef long long int Int64; +typedef unsigned long long int UInt64; +#define UINT64_CONST(n) n ## ULL +#endif + +#endif + +#ifdef _LZMA_NO_SYSTEM_SIZE_T +typedef UInt32 SizeT; +#else +typedef size_t SizeT; +#endif + +typedef int BoolInt; +/* typedef BoolInt Bool; */ +#define True 1 +#define False 0 + + +#ifdef _WIN32 +#define MY_STD_CALL __stdcall +#else +#define MY_STD_CALL +#endif + +#ifdef _MSC_VER + +#if _MSC_VER >= 1300 +#define MY_NO_INLINE __declspec(noinline) +#else +#define MY_NO_INLINE +#endif + +#define MY_FORCE_INLINE __forceinline + +#define MY_CDECL __cdecl +#define MY_FAST_CALL __fastcall + +#else // _MSC_VER + +#if (defined(__GNUC__) && (__GNUC__ >= 4)) \ + || (defined(__clang__) && (__clang_major__ >= 4)) \ + || defined(__INTEL_COMPILER) \ + || defined(__xlC__) +#define MY_NO_INLINE __attribute__((noinline)) +// #define MY_FORCE_INLINE __attribute__((always_inline)) inline +#else +#define MY_NO_INLINE +#endif + +#define MY_FORCE_INLINE + + +#define MY_CDECL + +#if defined(_M_IX86) \ + || defined(__i386__) +// #define MY_FAST_CALL __attribute__((fastcall)) +// #define MY_FAST_CALL __attribute__((cdecl)) +#define MY_FAST_CALL +#elif defined(MY_CPU_AMD64) +// #define MY_FAST_CALL __attribute__((ms_abi)) +#define MY_FAST_CALL +#else +#define MY_FAST_CALL +#endif + +#endif // _MSC_VER + + +/* The following interfaces use first parameter as pointer to structure */ + +typedef struct IByteIn IByteIn; +struct IByteIn +{ + Byte (*Read)(const IByteIn *p); /* reads one byte, returns 0 in case of EOF or error */ +}; +#define IByteIn_Read(p) (p)->Read(p) + + +typedef struct IByteOut IByteOut; +struct IByteOut +{ + void (*Write)(const IByteOut *p, Byte b); +}; +#define IByteOut_Write(p, b) (p)->Write(p, b) + + +typedef struct ISeqInStream ISeqInStream; +struct ISeqInStream +{ + SRes (*Read)(const ISeqInStream *p, void *buf, size_t *size); + /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. + (output(*size) < input(*size)) is allowed */ +}; +#define ISeqInStream_Read(p, buf, size) (p)->Read(p, buf, size) + +/* it can return SZ_ERROR_INPUT_EOF */ +SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size); +SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType); +SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf); + + +typedef struct ISeqOutStream ISeqOutStream; +struct ISeqOutStream +{ + size_t (*Write)(const ISeqOutStream *p, const void *buf, size_t size); + /* Returns: result - the number of actually written bytes. + (result < size) means error */ +}; +#define ISeqOutStream_Write(p, buf, size) (p)->Write(p, buf, size) + +typedef enum +{ + SZ_SEEK_SET = 0, + SZ_SEEK_CUR = 1, + SZ_SEEK_END = 2 +} ESzSeek; + + +typedef struct ISeekInStream ISeekInStream; +struct ISeekInStream +{ + SRes (*Read)(const ISeekInStream *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ + SRes (*Seek)(const ISeekInStream *p, Int64 *pos, ESzSeek origin); +}; +#define ISeekInStream_Read(p, buf, size) (p)->Read(p, buf, size) +#define ISeekInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin) + + +typedef struct ILookInStream ILookInStream; +struct ILookInStream +{ + SRes (*Look)(const ILookInStream *p, const void **buf, size_t *size); + /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. + (output(*size) > input(*size)) is not allowed + (output(*size) < input(*size)) is allowed */ + SRes (*Skip)(const ILookInStream *p, size_t offset); + /* offset must be <= output(*size) of Look */ + + SRes (*Read)(const ILookInStream *p, void *buf, size_t *size); + /* reads directly (without buffer). It's same as ISeqInStream::Read */ + SRes (*Seek)(const ILookInStream *p, Int64 *pos, ESzSeek origin); +}; + +#define ILookInStream_Look(p, buf, size) (p)->Look(p, buf, size) +#define ILookInStream_Skip(p, offset) (p)->Skip(p, offset) +#define ILookInStream_Read(p, buf, size) (p)->Read(p, buf, size) +#define ILookInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin) + + +SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size); +SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset); + +/* reads via ILookInStream::Read */ +SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType); +SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size); + + + +typedef struct +{ + ILookInStream vt; + const ISeekInStream *realStream; + + size_t pos; + size_t size; /* it's data size */ + + /* the following variables must be set outside */ + Byte *buf; + size_t bufSize; +} CLookToRead2; + +void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead); + +#define LookToRead2_Init(p) { (p)->pos = (p)->size = 0; } + + +typedef struct +{ + ISeqInStream vt; + const ILookInStream *realStream; +} CSecToLook; + +void SecToLook_CreateVTable(CSecToLook *p); + + + +typedef struct +{ + ISeqInStream vt; + const ILookInStream *realStream; +} CSecToRead; + +void SecToRead_CreateVTable(CSecToRead *p); + + +typedef struct ICompressProgress ICompressProgress; + +struct ICompressProgress +{ + SRes (*Progress)(const ICompressProgress *p, UInt64 inSize, UInt64 outSize); + /* Returns: result. (result != SZ_OK) means break. + Value (UInt64)(Int64)-1 for size means unknown value. */ +}; +#define ICompressProgress_Progress(p, inSize, outSize) (p)->Progress(p, inSize, outSize) + + + +typedef struct ISzAlloc ISzAlloc; +typedef const ISzAlloc * ISzAllocPtr; + +struct ISzAlloc +{ + void *(*Alloc)(ISzAllocPtr p, size_t size); + void (*Free)(ISzAllocPtr p, void *address); /* address can be 0 */ +}; + +#define ISzAlloc_Alloc(p, size) (p)->Alloc(p, size) +#define ISzAlloc_Free(p, a) (p)->Free(p, a) + +/* deprecated */ +#define IAlloc_Alloc(p, size) ISzAlloc_Alloc(p, size) +#define IAlloc_Free(p, a) ISzAlloc_Free(p, a) + + + + + +#ifndef MY_offsetof + #ifdef offsetof + #define MY_offsetof(type, m) offsetof(type, m) + /* + #define MY_offsetof(type, m) FIELD_OFFSET(type, m) + */ + #else + #define MY_offsetof(type, m) ((size_t)&(((type *)0)->m)) + #endif +#endif + + + +#ifndef MY_container_of + +/* +#define MY_container_of(ptr, type, m) container_of(ptr, type, m) +#define MY_container_of(ptr, type, m) CONTAINING_RECORD(ptr, type, m) +#define MY_container_of(ptr, type, m) ((type *)((char *)(ptr) - offsetof(type, m))) +#define MY_container_of(ptr, type, m) (&((type *)0)->m == (ptr), ((type *)(((char *)(ptr)) - MY_offsetof(type, m)))) +*/ + +/* + GCC shows warning: "perhaps the 'offsetof' macro was used incorrectly" + GCC 3.4.4 : classes with constructor + GCC 4.8.1 : classes with non-public variable members" +*/ + +#define MY_container_of(ptr, type, m) ((type *)(void *)((char *)(void *)(1 ? (ptr) : &((type *)0)->m) - MY_offsetof(type, m))) + +#endif + +#define CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) ((type *)(void *)(ptr)) + +/* +#define CONTAINER_FROM_VTBL(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) +*/ +#define CONTAINER_FROM_VTBL(ptr, type, m) MY_container_of(ptr, type, m) + +#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) +/* +#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL(ptr, type, m) +*/ + + +#define MY_memset_0_ARRAY(a) memset((a), 0, sizeof(a)) + +#ifdef _WIN32 + +#define CHAR_PATH_SEPARATOR '\\' +#define WCHAR_PATH_SEPARATOR L'\\' +#define STRING_PATH_SEPARATOR "\\" +#define WSTRING_PATH_SEPARATOR L"\\" + +#else + +#define CHAR_PATH_SEPARATOR '/' +#define WCHAR_PATH_SEPARATOR L'/' +#define STRING_PATH_SEPARATOR "/" +#define WSTRING_PATH_SEPARATOR L"/" + +#endif + +#define k_PropVar_TimePrec_0 0 +#define k_PropVar_TimePrec_Unix 1 +#define k_PropVar_TimePrec_DOS 2 +#define k_PropVar_TimePrec_HighPrec 3 +#define k_PropVar_TimePrec_Base 16 +#define k_PropVar_TimePrec_100ns (k_PropVar_TimePrec_Base + 7) +#define k_PropVar_TimePrec_1ns (k_PropVar_TimePrec_Base + 9) + +EXTERN_C_END + +#endif diff --git a/common/LZMA/SDK/C/7zVersion.h b/common/LZMA/SDK/C/7zVersion.h index 9d99c5d..49ea81d 100644 --- a/common/LZMA/SDK/C/7zVersion.h +++ b/common/LZMA/SDK/C/7zVersion.h @@ -1,7 +1,27 @@ -#define MY_VER_MAJOR 9 -#define MY_VER_MINOR 20 +#define MY_VER_MAJOR 22 +#define MY_VER_MINOR 01 #define MY_VER_BUILD 0 -#define MY_VERSION "9.20" -#define MY_DATE "2010-11-18" -#define MY_COPYRIGHT ": Igor Pavlov : Public domain" -#define MY_VERSION_COPYRIGHT_DATE MY_VERSION " " MY_COPYRIGHT " : " MY_DATE +#define MY_VERSION_NUMBERS "22.01" +#define MY_VERSION MY_VERSION_NUMBERS + +#ifdef MY_CPU_NAME + #define MY_VERSION_CPU MY_VERSION " (" MY_CPU_NAME ")" +#else + #define MY_VERSION_CPU MY_VERSION +#endif + +#define MY_DATE "2022-07-15" +#undef MY_COPYRIGHT +#undef MY_VERSION_COPYRIGHT_DATE +#define MY_AUTHOR_NAME "Igor Pavlov" +#define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2022 Igor Pavlov" + +#ifdef USE_COPYRIGHT_CR + #define MY_COPYRIGHT MY_COPYRIGHT_CR +#else + #define MY_COPYRIGHT MY_COPYRIGHT_PD +#endif + +#define MY_COPYRIGHT_DATE MY_COPYRIGHT " : " MY_DATE +#define MY_VERSION_COPYRIGHT_DATE MY_VERSION_CPU " : " MY_COPYRIGHT " : " MY_DATE diff --git a/common/LZMA/SDK/C/Bra.c b/common/LZMA/SDK/C/Bra.c new file mode 100644 index 0000000..3b854d9 --- /dev/null +++ b/common/LZMA/SDK/C/Bra.c @@ -0,0 +1,230 @@ +/* Bra.c -- Converters for RISC code +2021-02-09 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "CpuArch.h" +#include "Bra.h" + +SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +{ + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip += 4; + p = data; + lim = data + size; + + if (encoding) + + for (;;) + { + for (;;) + { + if (p >= lim) + return (SizeT)(p - data); + p += 4; + if (p[-1] == 0xEB) + break; + } + { + UInt32 v = GetUi32(p - 4); + v <<= 2; + v += ip + (UInt32)(p - data); + v >>= 2; + v &= 0x00FFFFFF; + v |= 0xEB000000; + SetUi32(p - 4, v); + } + } + + for (;;) + { + for (;;) + { + if (p >= lim) + return (SizeT)(p - data); + p += 4; + if (p[-1] == 0xEB) + break; + } + { + UInt32 v = GetUi32(p - 4); + v <<= 2; + v -= ip + (UInt32)(p - data); + v >>= 2; + v &= 0x00FFFFFF; + v |= 0xEB000000; + SetUi32(p - 4, v); + } + } +} + + +SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +{ + Byte *p; + const Byte *lim; + size &= ~(size_t)1; + p = data; + lim = data + size - 4; + + if (encoding) + + for (;;) + { + UInt32 b1; + for (;;) + { + UInt32 b3; + if (p > lim) + return (SizeT)(p - data); + b1 = p[1]; + b3 = p[3]; + p += 2; + b1 ^= 8; + if ((b3 & b1) >= 0xF8) + break; + } + { + UInt32 v = + ((UInt32)b1 << 19) + + (((UInt32)p[1] & 0x7) << 8) + + (((UInt32)p[-2] << 11)) + + (p[0]); + + p += 2; + { + UInt32 cur = (ip + (UInt32)(p - data)) >> 1; + v += cur; + } + + p[-4] = (Byte)(v >> 11); + p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); + p[-2] = (Byte)v; + p[-1] = (Byte)(0xF8 | (v >> 8)); + } + } + + for (;;) + { + UInt32 b1; + for (;;) + { + UInt32 b3; + if (p > lim) + return (SizeT)(p - data); + b1 = p[1]; + b3 = p[3]; + p += 2; + b1 ^= 8; + if ((b3 & b1) >= 0xF8) + break; + } + { + UInt32 v = + ((UInt32)b1 << 19) + + (((UInt32)p[1] & 0x7) << 8) + + (((UInt32)p[-2] << 11)) + + (p[0]); + + p += 2; + { + UInt32 cur = (ip + (UInt32)(p - data)) >> 1; + v -= cur; + } + + /* + SetUi16(p - 4, (UInt16)(((v >> 11) & 0x7FF) | 0xF000)); + SetUi16(p - 2, (UInt16)(v | 0xF800)); + */ + + p[-4] = (Byte)(v >> 11); + p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); + p[-2] = (Byte)v; + p[-1] = (Byte)(0xF8 | (v >> 8)); + } + } +} + + +SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +{ + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip -= 4; + p = data; + lim = data + size; + + for (;;) + { + for (;;) + { + if (p >= lim) + return (SizeT)(p - data); + p += 4; + /* if ((v & 0xFC000003) == 0x48000001) */ + if ((p[-4] & 0xFC) == 0x48 && (p[-1] & 3) == 1) + break; + } + { + UInt32 v = GetBe32(p - 4); + if (encoding) + v += ip + (UInt32)(p - data); + else + v -= ip + (UInt32)(p - data); + v &= 0x03FFFFFF; + v |= 0x48000000; + SetBe32(p - 4, v); + } + } +} + + +SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +{ + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip -= 4; + p = data; + lim = data + size; + + for (;;) + { + for (;;) + { + if (p >= lim) + return (SizeT)(p - data); + /* + v = GetBe32(p); + p += 4; + m = v + ((UInt32)5 << 29); + m ^= (UInt32)7 << 29; + m += (UInt32)1 << 22; + if ((m & ((UInt32)0x1FF << 23)) == 0) + break; + */ + p += 4; + if ((p[-4] == 0x40 && (p[-3] & 0xC0) == 0) || + (p[-4] == 0x7F && (p[-3] >= 0xC0))) + break; + } + { + UInt32 v = GetBe32(p - 4); + v <<= 2; + if (encoding) + v += ip + (UInt32)(p - data); + else + v -= ip + (UInt32)(p - data); + + v &= 0x01FFFFFF; + v -= (UInt32)1 << 24; + v ^= 0xFF000000; + v >>= 2; + v |= 0x40000000; + SetBe32(p - 4, v); + } + } +} diff --git a/common/LZMA/SDK/C/Bra.h b/common/LZMA/SDK/C/Bra.h index 5748c1c..855e37a 100644 --- a/common/LZMA/SDK/C/Bra.h +++ b/common/LZMA/SDK/C/Bra.h @@ -1,14 +1,12 @@ /* Bra.h -- Branch converters for executables -2009-02-07 : Igor Pavlov : Public domain */ +2013-01-18 : Igor Pavlov : Public domain */ #ifndef __BRA_H #define __BRA_H -#include "Types.h" +#include "7zTypes.h" -#ifdef __cplusplus -extern "C" { -#endif +EXTERN_C_BEGIN /* These functions convert relative addresses to absolute addresses @@ -61,8 +59,6 @@ SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -#ifdef __cplusplus -} -#endif +EXTERN_C_END #endif diff --git a/common/LZMA/SDK/C/Bra86.c b/common/LZMA/SDK/C/Bra86.c index 1ee0e70..10a0fbd 100644 --- a/common/LZMA/SDK/C/Bra86.c +++ b/common/LZMA/SDK/C/Bra86.c @@ -1,85 +1,82 @@ /* Bra86.c -- Converter for x86 code (BCJ) -2008-10-04 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ + +#include "Precomp.h" #include "Bra.h" -#define Test86MSByte(b) ((b) == 0 || (b) == 0xFF) - -const Byte kMaskToAllowedStatus[8] = {1, 1, 1, 0, 1, 0, 0, 0}; -const Byte kMaskToBitNumber[8] = {0, 1, 2, 2, 3, 3, 3, 3}; +#define Test86MSByte(b) ((((b) + 1) & 0xFE) == 0) SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding) { - SizeT bufferPos = 0, prevPosT; - UInt32 prevMask = *state & 0x7; + SizeT pos = 0; + UInt32 mask = *state & 7; if (size < 5) return 0; + size -= 4; ip += 5; - prevPosT = (SizeT)0 - 1; for (;;) { - Byte *p = data + bufferPos; - Byte *limit = data + size - 4; + Byte *p = data + pos; + const Byte *limit = data + size; for (; p < limit; p++) if ((*p & 0xFE) == 0xE8) break; - bufferPos = (SizeT)(p - data); - if (p >= limit) - break; - prevPosT = bufferPos - prevPosT; - if (prevPosT > 3) - prevMask = 0; - else + { - prevMask = (prevMask << ((int)prevPosT - 1)) & 0x7; - if (prevMask != 0) + SizeT d = (SizeT)(p - data) - pos; + pos = (SizeT)(p - data); + if (p >= limit) { - Byte b = p[4 - kMaskToBitNumber[prevMask]]; - if (!kMaskToAllowedStatus[prevMask] || Test86MSByte(b)) + *state = (d > 2 ? 0 : mask >> (unsigned)d); + return pos; + } + if (d > 2) + mask = 0; + else + { + mask >>= (unsigned)d; + if (mask != 0 && (mask > 4 || mask == 3 || Test86MSByte(p[(size_t)(mask >> 1) + 1]))) { - prevPosT = bufferPos; - prevMask = ((prevMask << 1) & 0x7) | 1; - bufferPos++; + mask = (mask >> 1) | 4; + pos++; continue; } } } - prevPosT = bufferPos; if (Test86MSByte(p[4])) { - UInt32 src = ((UInt32)p[4] << 24) | ((UInt32)p[3] << 16) | ((UInt32)p[2] << 8) | ((UInt32)p[1]); - UInt32 dest; - for (;;) + UInt32 v = ((UInt32)p[4] << 24) | ((UInt32)p[3] << 16) | ((UInt32)p[2] << 8) | ((UInt32)p[1]); + UInt32 cur = ip + (UInt32)pos; + pos += 5; + if (encoding) + v += cur; + else + v -= cur; + if (mask != 0) { - Byte b; - int index; - if (encoding) - dest = (ip + (UInt32)bufferPos) + src; - else - dest = src - (ip + (UInt32)bufferPos); - if (prevMask == 0) - break; - index = kMaskToBitNumber[prevMask] * 8; - b = (Byte)(dest >> (24 - index)); - if (!Test86MSByte(b)) - break; - src = dest ^ ((1 << (32 - index)) - 1); + unsigned sh = (mask & 6) << 2; + if (Test86MSByte((Byte)(v >> sh))) + { + v ^= (((UInt32)0x100 << sh) - 1); + if (encoding) + v += cur; + else + v -= cur; + } + mask = 0; } - p[4] = (Byte)(~(((dest >> 24) & 1) - 1)); - p[3] = (Byte)(dest >> 16); - p[2] = (Byte)(dest >> 8); - p[1] = (Byte)dest; - bufferPos += 5; + p[1] = (Byte)v; + p[2] = (Byte)(v >> 8); + p[3] = (Byte)(v >> 16); + p[4] = (Byte)(0 - ((v >> 24) & 1)); } else { - prevMask = ((prevMask << 1) & 0x7) | 1; - bufferPos++; + mask = (mask >> 1) | 4; + pos++; } } - prevPosT = bufferPos - prevPosT; - *state = ((prevPosT > 3) ? 0 : ((prevMask << ((int)prevPosT - 1)) & 0x7)); - return bufferPos; } diff --git a/common/LZMA/SDK/C/Compiler.h b/common/LZMA/SDK/C/Compiler.h new file mode 100644 index 0000000..a9816fa --- /dev/null +++ b/common/LZMA/SDK/C/Compiler.h @@ -0,0 +1,43 @@ +/* Compiler.h +2021-01-05 : Igor Pavlov : Public domain */ + +#ifndef __7Z_COMPILER_H +#define __7Z_COMPILER_H + + #ifdef __clang__ + #pragma clang diagnostic ignored "-Wunused-private-field" + #endif + +#ifdef _MSC_VER + + #ifdef UNDER_CE + #define RPC_NO_WINDOWS_H + /* #pragma warning(disable : 4115) // '_RPC_ASYNC_STATE' : named type definition in parentheses */ + #pragma warning(disable : 4201) // nonstandard extension used : nameless struct/union + #pragma warning(disable : 4214) // nonstandard extension used : bit field types other than int + #endif + + #if _MSC_VER >= 1300 + #pragma warning(disable : 4996) // This function or variable may be unsafe + #else + #pragma warning(disable : 4511) // copy constructor could not be generated + #pragma warning(disable : 4512) // assignment operator could not be generated + #pragma warning(disable : 4514) // unreferenced inline function has been removed + #pragma warning(disable : 4702) // unreachable code + #pragma warning(disable : 4710) // not inlined + #pragma warning(disable : 4714) // function marked as __forceinline not inlined + #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information + #endif + + #ifdef __clang__ + #pragma clang diagnostic ignored "-Wdeprecated-declarations" + #pragma clang diagnostic ignored "-Wmicrosoft-exception-spec" + // #pragma clang diagnostic ignored "-Wreserved-id-macro" + #endif + +#endif + +#define UNUSED_VAR(x) (void)x; +/* #define UNUSED_VAR(x) x=x; */ + +#endif diff --git a/common/LZMA/SDK/C/CpuArch.c b/common/LZMA/SDK/C/CpuArch.c new file mode 100644 index 0000000..fa9afe3 --- /dev/null +++ b/common/LZMA/SDK/C/CpuArch.c @@ -0,0 +1,478 @@ +/* CpuArch.c -- CPU specific code +2021-07-13 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "CpuArch.h" + +#ifdef MY_CPU_X86_OR_AMD64 + +#if (defined(_MSC_VER) && !defined(MY_CPU_AMD64)) || defined(__GNUC__) +#define USE_ASM +#endif + +#if !defined(USE_ASM) && _MSC_VER >= 1500 +#include +#endif + +#if defined(USE_ASM) && !defined(MY_CPU_AMD64) +static UInt32 CheckFlag(UInt32 flag) +{ + #ifdef _MSC_VER + __asm pushfd; + __asm pop EAX; + __asm mov EDX, EAX; + __asm xor EAX, flag; + __asm push EAX; + __asm popfd; + __asm pushfd; + __asm pop EAX; + __asm xor EAX, EDX; + __asm push EDX; + __asm popfd; + __asm and flag, EAX; + #else + __asm__ __volatile__ ( + "pushf\n\t" + "pop %%EAX\n\t" + "movl %%EAX,%%EDX\n\t" + "xorl %0,%%EAX\n\t" + "push %%EAX\n\t" + "popf\n\t" + "pushf\n\t" + "pop %%EAX\n\t" + "xorl %%EDX,%%EAX\n\t" + "push %%EDX\n\t" + "popf\n\t" + "andl %%EAX, %0\n\t": + "=c" (flag) : "c" (flag) : + "%eax", "%edx"); + #endif + return flag; +} +#define CHECK_CPUID_IS_SUPPORTED if (CheckFlag(1 << 18) == 0 || CheckFlag(1 << 21) == 0) return False; +#else +#define CHECK_CPUID_IS_SUPPORTED +#endif + +#ifndef USE_ASM + #ifdef _MSC_VER + #if _MSC_VER >= 1600 + #define MY__cpuidex __cpuidex + #else + +/* + __cpuid (function == 4) requires subfunction number in ECX. + MSDN: The __cpuid intrinsic clears the ECX register before calling the cpuid instruction. + __cpuid() in new MSVC clears ECX. + __cpuid() in old MSVC (14.00) doesn't clear ECX + We still can use __cpuid for low (function) values that don't require ECX, + but __cpuid() in old MSVC will be incorrect for some function values: (function == 4). + So here we use the hack for old MSVC to send (subFunction) in ECX register to cpuid instruction, + where ECX value is first parameter for FAST_CALL / NO_INLINE function, + So the caller of MY__cpuidex_HACK() sets ECX as subFunction, and + old MSVC for __cpuid() doesn't change ECX and cpuid instruction gets (subFunction) value. + + DON'T remove MY_NO_INLINE and MY_FAST_CALL for MY__cpuidex_HACK() !!! +*/ + +static +MY_NO_INLINE +void MY_FAST_CALL MY__cpuidex_HACK(UInt32 subFunction, int *CPUInfo, UInt32 function) +{ + UNUSED_VAR(subFunction); + __cpuid(CPUInfo, function); +} + + #define MY__cpuidex(info, func, func2) MY__cpuidex_HACK(func2, info, func) + #pragma message("======== MY__cpuidex_HACK WAS USED ========") + #endif + #else + #define MY__cpuidex(info, func, func2) __cpuid(info, func) + #pragma message("======== (INCORRECT ?) cpuid WAS USED ========") + #endif +#endif + + + + +void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) +{ + #ifdef USE_ASM + + #ifdef _MSC_VER + + UInt32 a2, b2, c2, d2; + __asm xor EBX, EBX; + __asm xor ECX, ECX; + __asm xor EDX, EDX; + __asm mov EAX, function; + __asm cpuid; + __asm mov a2, EAX; + __asm mov b2, EBX; + __asm mov c2, ECX; + __asm mov d2, EDX; + + *a = a2; + *b = b2; + *c = c2; + *d = d2; + + #else + + __asm__ __volatile__ ( + #if defined(MY_CPU_AMD64) && defined(__PIC__) + "mov %%rbx, %%rdi;" + "cpuid;" + "xchg %%rbx, %%rdi;" + : "=a" (*a) , + "=D" (*b) , + #elif defined(MY_CPU_X86) && defined(__PIC__) + "mov %%ebx, %%edi;" + "cpuid;" + "xchgl %%ebx, %%edi;" + : "=a" (*a) , + "=D" (*b) , + #else + "cpuid" + : "=a" (*a) , + "=b" (*b) , + #endif + "=c" (*c) , + "=d" (*d) + : "0" (function), "c"(0) ) ; + + #endif + + #else + + int CPUInfo[4]; + + MY__cpuidex(CPUInfo, (int)function, 0); + + *a = (UInt32)CPUInfo[0]; + *b = (UInt32)CPUInfo[1]; + *c = (UInt32)CPUInfo[2]; + *d = (UInt32)CPUInfo[3]; + + #endif +} + +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p) +{ + CHECK_CPUID_IS_SUPPORTED + MyCPUID(0, &p->maxFunc, &p->vendor[0], &p->vendor[2], &p->vendor[1]); + MyCPUID(1, &p->ver, &p->b, &p->c, &p->d); + return True; +} + +static const UInt32 kVendors[][3] = +{ + { 0x756E6547, 0x49656E69, 0x6C65746E}, + { 0x68747541, 0x69746E65, 0x444D4163}, + { 0x746E6543, 0x48727561, 0x736C7561} +}; + +int x86cpuid_GetFirm(const Cx86cpuid *p) +{ + unsigned i; + for (i = 0; i < sizeof(kVendors) / sizeof(kVendors[i]); i++) + { + const UInt32 *v = kVendors[i]; + if (v[0] == p->vendor[0] && + v[1] == p->vendor[1] && + v[2] == p->vendor[2]) + return (int)i; + } + return -1; +} + +BoolInt CPU_Is_InOrder() +{ + Cx86cpuid p; + int firm; + UInt32 family, model; + if (!x86cpuid_CheckAndRead(&p)) + return True; + + family = x86cpuid_GetFamily(p.ver); + model = x86cpuid_GetModel(p.ver); + + firm = x86cpuid_GetFirm(&p); + + switch (firm) + { + case CPU_FIRM_INTEL: return (family < 6 || (family == 6 && ( + /* In-Order Atom CPU */ + model == 0x1C /* 45 nm, N4xx, D4xx, N5xx, D5xx, 230, 330 */ + || model == 0x26 /* 45 nm, Z6xx */ + || model == 0x27 /* 32 nm, Z2460 */ + || model == 0x35 /* 32 nm, Z2760 */ + || model == 0x36 /* 32 nm, N2xxx, D2xxx */ + ))); + case CPU_FIRM_AMD: return (family < 5 || (family == 5 && (model < 6 || model == 0xA))); + case CPU_FIRM_VIA: return (family < 6 || (family == 6 && model < 0xF)); + } + return True; +} + +#if !defined(MY_CPU_AMD64) && defined(_WIN32) +#include +static BoolInt CPU_Sys_Is_SSE_Supported() +{ + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + if (!GetVersionEx(&vi)) + return False; + return (vi.dwMajorVersion >= 5); +} +#define CHECK_SYS_SSE_SUPPORT if (!CPU_Sys_Is_SSE_Supported()) return False; +#else +#define CHECK_SYS_SSE_SUPPORT +#endif + + +static UInt32 X86_CPUID_ECX_Get_Flags() +{ + Cx86cpuid p; + CHECK_SYS_SSE_SUPPORT + if (!x86cpuid_CheckAndRead(&p)) + return 0; + return p.c; +} + +BoolInt CPU_IsSupported_AES() +{ + return (X86_CPUID_ECX_Get_Flags() >> 25) & 1; +} + +BoolInt CPU_IsSupported_SSSE3() +{ + return (X86_CPUID_ECX_Get_Flags() >> 9) & 1; +} + +BoolInt CPU_IsSupported_SSE41() +{ + return (X86_CPUID_ECX_Get_Flags() >> 19) & 1; +} + +BoolInt CPU_IsSupported_SHA() +{ + Cx86cpuid p; + CHECK_SYS_SSE_SUPPORT + if (!x86cpuid_CheckAndRead(&p)) + return False; + + if (p.maxFunc < 7) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); + return (d[1] >> 29) & 1; + } +} + +// #include + +#ifdef _WIN32 +#include +#endif + +BoolInt CPU_IsSupported_AVX2() +{ + Cx86cpuid p; + CHECK_SYS_SSE_SUPPORT + + #ifdef _WIN32 + #define MY__PF_XSAVE_ENABLED 17 + if (!IsProcessorFeaturePresent(MY__PF_XSAVE_ENABLED)) + return False; + #endif + + if (!x86cpuid_CheckAndRead(&p)) + return False; + if (p.maxFunc < 7) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); + // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); + return 1 + & (d[1] >> 5); // avx2 + } +} + +BoolInt CPU_IsSupported_VAES_AVX2() +{ + Cx86cpuid p; + CHECK_SYS_SSE_SUPPORT + + #ifdef _WIN32 + #define MY__PF_XSAVE_ENABLED 17 + if (!IsProcessorFeaturePresent(MY__PF_XSAVE_ENABLED)) + return False; + #endif + + if (!x86cpuid_CheckAndRead(&p)) + return False; + if (p.maxFunc < 7) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); + // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); + return 1 + & (d[1] >> 5) // avx2 + // & (d[1] >> 31) // avx512vl + & (d[2] >> 9); // vaes // VEX-256/EVEX + } +} + +BoolInt CPU_IsSupported_PageGB() +{ + Cx86cpuid cpuid; + if (!x86cpuid_CheckAndRead(&cpuid)) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000000, &d[0], &d[1], &d[2], &d[3]); + if (d[0] < 0x80000001) + return False; + } + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000001, &d[0], &d[1], &d[2], &d[3]); + return (d[3] >> 26) & 1; + } +} + + +#elif defined(MY_CPU_ARM_OR_ARM64) + +#ifdef _WIN32 + +#include + +BoolInt CPU_IsSupported_CRC32() { return IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } +BoolInt CPU_IsSupported_CRYPTO() { return IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } +BoolInt CPU_IsSupported_NEON() { return IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } + +#else + +#if defined(__APPLE__) + +/* +#include +#include +static void Print_sysctlbyname(const char *name) +{ + size_t bufSize = 256; + char buf[256]; + int res = sysctlbyname(name, &buf, &bufSize, NULL, 0); + { + int i; + printf("\nres = %d : %s : '%s' : bufSize = %d, numeric", res, name, buf, (unsigned)bufSize); + for (i = 0; i < 20; i++) + printf(" %2x", (unsigned)(Byte)buf[i]); + + } +} +*/ + +static BoolInt My_sysctlbyname_Get_BoolInt(const char *name) +{ + UInt32 val = 0; + if (My_sysctlbyname_Get_UInt32(name, &val) == 0 && val == 1) + return 1; + return 0; +} + + /* + Print_sysctlbyname("hw.pagesize"); + Print_sysctlbyname("machdep.cpu.brand_string"); + */ + +BoolInt CPU_IsSupported_CRC32(void) +{ + return My_sysctlbyname_Get_BoolInt("hw.optional.armv8_crc32"); +} + +BoolInt CPU_IsSupported_NEON(void) +{ + return My_sysctlbyname_Get_BoolInt("hw.optional.neon"); +} + +#ifdef MY_CPU_ARM64 +#define APPLE_CRYPTO_SUPPORT_VAL 1 +#else +#define APPLE_CRYPTO_SUPPORT_VAL 0 +#endif + +BoolInt CPU_IsSupported_SHA1(void) { return APPLE_CRYPTO_SUPPORT_VAL; } +BoolInt CPU_IsSupported_SHA2(void) { return APPLE_CRYPTO_SUPPORT_VAL; } +BoolInt CPU_IsSupported_AES (void) { return APPLE_CRYPTO_SUPPORT_VAL; } + + +#else // __APPLE__ + +#include + +#define USE_HWCAP + +#ifdef USE_HWCAP + +#include + + #define MY_HWCAP_CHECK_FUNC_2(name1, name2) \ + BoolInt CPU_IsSupported_ ## name1() { return (getauxval(AT_HWCAP) & (HWCAP_ ## name2)) ? 1 : 0; } + +#ifdef MY_CPU_ARM64 + #define MY_HWCAP_CHECK_FUNC(name) \ + MY_HWCAP_CHECK_FUNC_2(name, name) + MY_HWCAP_CHECK_FUNC_2(NEON, ASIMD) +// MY_HWCAP_CHECK_FUNC (ASIMD) +#elif defined(MY_CPU_ARM) + #define MY_HWCAP_CHECK_FUNC(name) \ + BoolInt CPU_IsSupported_ ## name() { return (getauxval(AT_HWCAP2) & (HWCAP2_ ## name)) ? 1 : 0; } + MY_HWCAP_CHECK_FUNC_2(NEON, NEON) +#endif + +#else // USE_HWCAP + + #define MY_HWCAP_CHECK_FUNC(name) \ + BoolInt CPU_IsSupported_ ## name() { return 0; } + MY_HWCAP_CHECK_FUNC(NEON) + +#endif // USE_HWCAP + +MY_HWCAP_CHECK_FUNC (CRC32) +MY_HWCAP_CHECK_FUNC (SHA1) +MY_HWCAP_CHECK_FUNC (SHA2) +MY_HWCAP_CHECK_FUNC (AES) + +#endif // __APPLE__ +#endif // _WIN32 + +#endif // MY_CPU_ARM_OR_ARM64 + + + +#ifdef __APPLE__ + +#include + +int My_sysctlbyname_Get(const char *name, void *buf, size_t *bufSize) +{ + return sysctlbyname(name, buf, bufSize, NULL, 0); +} + +int My_sysctlbyname_Get_UInt32(const char *name, UInt32 *val) +{ + size_t bufSize = sizeof(*val); + int res = My_sysctlbyname_Get(name, val, &bufSize); + if (res == 0 && bufSize != sizeof(*val)) + return EFAULT; + return res; +} + +#endif diff --git a/common/LZMA/SDK/C/CpuArch.h b/common/LZMA/SDK/C/CpuArch.h index 01930c7..4856fbb 100644 --- a/common/LZMA/SDK/C/CpuArch.h +++ b/common/LZMA/SDK/C/CpuArch.h @@ -1,77 +1,290 @@ /* CpuArch.h -- CPU specific code -2010-10-26: Igor Pavlov : Public domain */ +2022-07-15 : Igor Pavlov : Public domain */ #ifndef __CPU_ARCH_H #define __CPU_ARCH_H -#include "Types.h" +#include "7zTypes.h" EXTERN_C_BEGIN /* MY_CPU_LE means that CPU is LITTLE ENDIAN. -If MY_CPU_LE is not defined, we don't know about that property of platform (it can be LITTLE ENDIAN). +MY_CPU_BE means that CPU is BIG ENDIAN. +If MY_CPU_LE and MY_CPU_BE are not defined, we don't know about ENDIANNESS of platform. MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned memory accesses. -If MY_CPU_LE_UNALIGN is not defined, we don't know about these properties of platform. + +MY_CPU_64BIT means that processor can work with 64-bit registers. + MY_CPU_64BIT can be used to select fast code branch + MY_CPU_64BIT doesn't mean that (sizeof(void *) == 8) */ -#if defined(_M_X64) || defined(_M_AMD64) || defined(__x86_64__) -#define MY_CPU_AMD64 +#if defined(_M_X64) \ + || defined(_M_AMD64) \ + || defined(__x86_64__) \ + || defined(__AMD64__) \ + || defined(__amd64__) + #define MY_CPU_AMD64 + #ifdef __ILP32__ + #define MY_CPU_NAME "x32" + #define MY_CPU_SIZEOF_POINTER 4 + #else + #define MY_CPU_NAME "x64" + #define MY_CPU_SIZEOF_POINTER 8 + #endif + #define MY_CPU_64BIT #endif -#if defined(MY_CPU_AMD64) || defined(_M_IA64) -#define MY_CPU_64BIT + +#if defined(_M_IX86) \ + || defined(__i386__) + #define MY_CPU_X86 + #define MY_CPU_NAME "x86" + /* #define MY_CPU_32BIT */ + #define MY_CPU_SIZEOF_POINTER 4 #endif -#if defined(_M_IX86) || defined(__i386__) -#define MY_CPU_X86 + +#if defined(_M_ARM64) \ + || defined(__AARCH64EL__) \ + || defined(__AARCH64EB__) \ + || defined(__aarch64__) + #define MY_CPU_ARM64 + #define MY_CPU_NAME "arm64" + #define MY_CPU_64BIT #endif + +#if defined(_M_ARM) \ + || defined(_M_ARM_NT) \ + || defined(_M_ARMT) \ + || defined(__arm__) \ + || defined(__thumb__) \ + || defined(__ARMEL__) \ + || defined(__ARMEB__) \ + || defined(__THUMBEL__) \ + || defined(__THUMBEB__) + #define MY_CPU_ARM + + #if defined(__thumb__) || defined(__THUMBEL__) || defined(_M_ARMT) + #define MY_CPU_NAME "armt" + #else + #define MY_CPU_NAME "arm" + #endif + /* #define MY_CPU_32BIT */ + #define MY_CPU_SIZEOF_POINTER 4 +#endif + + +#if defined(_M_IA64) \ + || defined(__ia64__) + #define MY_CPU_IA64 + #define MY_CPU_NAME "ia64" + #define MY_CPU_64BIT +#endif + + +#if defined(__mips64) \ + || defined(__mips64__) \ + || (defined(__mips) && (__mips == 64 || __mips == 4 || __mips == 3)) + #define MY_CPU_NAME "mips64" + #define MY_CPU_64BIT +#elif defined(__mips__) + #define MY_CPU_NAME "mips" + /* #define MY_CPU_32BIT */ +#endif + + +#if defined(__ppc64__) \ + || defined(__powerpc64__) \ + || defined(__ppc__) \ + || defined(__powerpc__) \ + || defined(__PPC__) \ + || defined(_POWER) + +#if defined(__ppc64__) \ + || defined(__powerpc64__) \ + || defined(_LP64) \ + || defined(__64BIT__) + #ifdef __ILP32__ + #define MY_CPU_NAME "ppc64-32" + #define MY_CPU_SIZEOF_POINTER 4 + #else + #define MY_CPU_NAME "ppc64" + #define MY_CPU_SIZEOF_POINTER 8 + #endif + #define MY_CPU_64BIT +#else + #define MY_CPU_NAME "ppc" + #define MY_CPU_SIZEOF_POINTER 4 + /* #define MY_CPU_32BIT */ +#endif +#endif + + +#if defined(__riscv) \ + || defined(__riscv__) + #if __riscv_xlen == 32 + #define MY_CPU_NAME "riscv32" + #elif __riscv_xlen == 64 + #define MY_CPU_NAME "riscv64" + #else + #define MY_CPU_NAME "riscv" + #endif +#endif + + #if defined(MY_CPU_X86) || defined(MY_CPU_AMD64) #define MY_CPU_X86_OR_AMD64 #endif -#if defined(MY_CPU_X86) || defined(_M_ARM) -#define MY_CPU_32BIT +#if defined(MY_CPU_ARM) || defined(MY_CPU_ARM64) +#define MY_CPU_ARM_OR_ARM64 #endif -#if defined(_WIN32) && defined(_M_ARM) -#define MY_CPU_ARM_LE + +#ifdef _WIN32 + + #ifdef MY_CPU_ARM + #define MY_CPU_ARM_LE + #endif + + #ifdef MY_CPU_ARM64 + #define MY_CPU_ARM64_LE + #endif + + #ifdef _M_IA64 + #define MY_CPU_IA64_LE + #endif + #endif -#if defined(_WIN32) && defined(_M_IA64) -#define MY_CPU_IA64_LE + +#if defined(MY_CPU_X86_OR_AMD64) \ + || defined(MY_CPU_ARM_LE) \ + || defined(MY_CPU_ARM64_LE) \ + || defined(MY_CPU_IA64_LE) \ + || defined(__LITTLE_ENDIAN__) \ + || defined(__ARMEL__) \ + || defined(__THUMBEL__) \ + || defined(__AARCH64EL__) \ + || defined(__MIPSEL__) \ + || defined(__MIPSEL) \ + || defined(_MIPSEL) \ + || defined(__BFIN__) \ + || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + #define MY_CPU_LE #endif -#if defined(MY_CPU_X86_OR_AMD64) -#define MY_CPU_LE_UNALIGN +#if defined(__BIG_ENDIAN__) \ + || defined(__ARMEB__) \ + || defined(__THUMBEB__) \ + || defined(__AARCH64EB__) \ + || defined(__MIPSEB__) \ + || defined(__MIPSEB) \ + || defined(_MIPSEB) \ + || defined(__m68k__) \ + || defined(__s390__) \ + || defined(__s390x__) \ + || defined(__zarch__) \ + || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) + #define MY_CPU_BE #endif -#if defined(MY_CPU_X86_OR_AMD64) || defined(MY_CPU_ARM_LE) || defined(MY_CPU_IA64_LE) || defined(__ARMEL__) || defined(__MIPSEL__) || defined(__LITTLE_ENDIAN__) -#define MY_CPU_LE -#endif - -#if defined(__BIG_ENDIAN__) -#define MY_CPU_BE -#endif #if defined(MY_CPU_LE) && defined(MY_CPU_BE) -Stop_Compiling_Bad_Endian + #error Stop_Compiling_Bad_Endian #endif + +#if defined(MY_CPU_32BIT) && defined(MY_CPU_64BIT) + #error Stop_Compiling_Bad_32_64_BIT +#endif + +#ifdef __SIZEOF_POINTER__ + #ifdef MY_CPU_SIZEOF_POINTER + #if MY_CPU_SIZEOF_POINTER != __SIZEOF_POINTER__ + #error Stop_Compiling_Bad_MY_CPU_PTR_SIZE + #endif + #else + #define MY_CPU_SIZEOF_POINTER __SIZEOF_POINTER__ + #endif +#endif + +#if defined(MY_CPU_SIZEOF_POINTER) && (MY_CPU_SIZEOF_POINTER == 4) +#if defined (_LP64) + #error Stop_Compiling_Bad_MY_CPU_PTR_SIZE +#endif +#endif + +#ifdef _MSC_VER + #if _MSC_VER >= 1300 + #define MY_CPU_pragma_pack_push_1 __pragma(pack(push, 1)) + #define MY_CPU_pragma_pop __pragma(pack(pop)) + #else + #define MY_CPU_pragma_pack_push_1 + #define MY_CPU_pragma_pop + #endif +#else + #ifdef __xlC__ + #define MY_CPU_pragma_pack_push_1 _Pragma("pack(1)") + #define MY_CPU_pragma_pop _Pragma("pack()") + #else + #define MY_CPU_pragma_pack_push_1 _Pragma("pack(push, 1)") + #define MY_CPU_pragma_pop _Pragma("pack(pop)") + #endif +#endif + + +#ifndef MY_CPU_NAME + #ifdef MY_CPU_LE + #define MY_CPU_NAME "LE" + #elif defined(MY_CPU_BE) + #define MY_CPU_NAME "BE" + #else + /* + #define MY_CPU_NAME "" + */ + #endif +#endif + + + + + +#ifdef MY_CPU_LE + #if defined(MY_CPU_X86_OR_AMD64) \ + || defined(MY_CPU_ARM64) + #define MY_CPU_LE_UNALIGN + #define MY_CPU_LE_UNALIGN_64 + #elif defined(__ARM_FEATURE_UNALIGNED) + /* gcc9 for 32-bit arm can use LDRD instruction that requires 32-bit alignment. + So we can't use unaligned 64-bit operations. */ + #define MY_CPU_LE_UNALIGN + #endif +#endif + + #ifdef MY_CPU_LE_UNALIGN -#define GetUi16(p) (*(const UInt16 *)(p)) -#define GetUi32(p) (*(const UInt32 *)(p)) -#define GetUi64(p) (*(const UInt64 *)(p)) -#define SetUi16(p, d) *(UInt16 *)(p) = (d); -#define SetUi32(p, d) *(UInt32 *)(p) = (d); -#define SetUi64(p, d) *(UInt64 *)(p) = (d); +#define GetUi16(p) (*(const UInt16 *)(const void *)(p)) +#define GetUi32(p) (*(const UInt32 *)(const void *)(p)) +#ifdef MY_CPU_LE_UNALIGN_64 +#define GetUi64(p) (*(const UInt64 *)(const void *)(p)) +#endif + +#define SetUi16(p, v) { *(UInt16 *)(void *)(p) = (v); } +#define SetUi32(p, v) { *(UInt32 *)(void *)(p) = (v); } +#ifdef MY_CPU_LE_UNALIGN_64 +#define SetUi64(p, v) { *(UInt64 *)(void *)(p) = (v); } +#endif #else -#define GetUi16(p) (((const Byte *)(p))[0] | ((UInt16)((const Byte *)(p))[1] << 8)) +#define GetUi16(p) ( (UInt16) ( \ + ((const Byte *)(p))[0] | \ + ((UInt16)((const Byte *)(p))[1] << 8) )) #define GetUi32(p) ( \ ((const Byte *)(p))[0] | \ @@ -79,30 +292,63 @@ Stop_Compiling_Bad_Endian ((UInt32)((const Byte *)(p))[2] << 16) | \ ((UInt32)((const Byte *)(p))[3] << 24)) -#define GetUi64(p) (GetUi32(p) | ((UInt64)GetUi32(((const Byte *)(p)) + 4) << 32)) +#define SetUi16(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ + _ppp_[0] = (Byte)_vvv_; \ + _ppp_[1] = (Byte)(_vvv_ >> 8); } -#define SetUi16(p, d) { UInt32 _x_ = (d); \ - ((Byte *)(p))[0] = (Byte)_x_; \ - ((Byte *)(p))[1] = (Byte)(_x_ >> 8); } - -#define SetUi32(p, d) { UInt32 _x_ = (d); \ - ((Byte *)(p))[0] = (Byte)_x_; \ - ((Byte *)(p))[1] = (Byte)(_x_ >> 8); \ - ((Byte *)(p))[2] = (Byte)(_x_ >> 16); \ - ((Byte *)(p))[3] = (Byte)(_x_ >> 24); } - -#define SetUi64(p, d) { UInt64 _x64_ = (d); \ - SetUi32(p, (UInt32)_x64_); \ - SetUi32(((Byte *)(p)) + 4, (UInt32)(_x64_ >> 32)); } +#define SetUi32(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ + _ppp_[0] = (Byte)_vvv_; \ + _ppp_[1] = (Byte)(_vvv_ >> 8); \ + _ppp_[2] = (Byte)(_vvv_ >> 16); \ + _ppp_[3] = (Byte)(_vvv_ >> 24); } #endif -#if defined(MY_CPU_LE_UNALIGN) && defined(_WIN64) && (_MSC_VER >= 1300) +#ifndef MY_CPU_LE_UNALIGN_64 + +#define GetUi64(p) (GetUi32(p) | ((UInt64)GetUi32(((const Byte *)(p)) + 4) << 32)) + +#define SetUi64(p, v) { Byte *_ppp2_ = (Byte *)(p); UInt64 _vvv2_ = (v); \ + SetUi32(_ppp2_ , (UInt32)_vvv2_); \ + SetUi32(_ppp2_ + 4, (UInt32)(_vvv2_ >> 32)); } + +#endif + + + + +#ifdef __has_builtin + #define MY__has_builtin(x) __has_builtin(x) +#else + #define MY__has_builtin(x) 0 +#endif + +#if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ defined(_MSC_VER) && (_MSC_VER >= 1300) + +/* Note: we use bswap instruction, that is unsupported in 386 cpu */ + +#include + +#pragma intrinsic(_byteswap_ushort) #pragma intrinsic(_byteswap_ulong) #pragma intrinsic(_byteswap_uint64) -#define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p)) -#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p)) + +/* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */ +#define GetBe32(p) _byteswap_ulong (*(const UInt32 *)(const void *)(p)) +#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const void *)(p)) + +#define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v) + +#elif defined(MY_CPU_LE_UNALIGN) && ( \ + (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ + || (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) ) + +/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const void *)(p)) */ +#define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const void *)(p)) +#define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const void *)(p)) + +#define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = __builtin_bswap32(v) #else @@ -114,9 +360,23 @@ Stop_Compiling_Bad_Endian #define GetBe64(p) (((UInt64)GetBe32(p) << 32) | GetBe32(((const Byte *)(p)) + 4)) +#define SetBe32(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ + _ppp_[0] = (Byte)(_vvv_ >> 24); \ + _ppp_[1] = (Byte)(_vvv_ >> 16); \ + _ppp_[2] = (Byte)(_vvv_ >> 8); \ + _ppp_[3] = (Byte)_vvv_; } + +#endif + + +#ifndef GetBe16 + +#define GetBe16(p) ( (UInt16) ( \ + ((UInt16)((const Byte *)(p))[0] << 8) | \ + ((const Byte *)(p))[1] )) + #endif -#define GetBe16(p) (((UInt16)((const Byte *)(p))[0] << 8) | ((const Byte *)(p))[1]) #ifdef MY_CPU_X86_OR_AMD64 @@ -138,16 +398,46 @@ enum CPU_FIRM_VIA }; -Bool x86cpuid_CheckAndRead(Cx86cpuid *p); +void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d); + +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p); int x86cpuid_GetFirm(const Cx86cpuid *p); -#define x86cpuid_GetFamily(p) (((p)->ver >> 8) & 0xFF00F) -#define x86cpuid_GetModel(p) (((p)->ver >> 4) & 0xF00F) -#define x86cpuid_GetStepping(p) ((p)->ver & 0xF) +#define x86cpuid_GetFamily(ver) (((ver >> 16) & 0xFF0) | ((ver >> 8) & 0xF)) +#define x86cpuid_GetModel(ver) (((ver >> 12) & 0xF0) | ((ver >> 4) & 0xF)) +#define x86cpuid_GetStepping(ver) (ver & 0xF) -Bool CPU_Is_InOrder(); -Bool CPU_Is_Aes_Supported(); +BoolInt CPU_Is_InOrder(void); +BoolInt CPU_IsSupported_AES(void); +BoolInt CPU_IsSupported_AVX2(void); +BoolInt CPU_IsSupported_VAES_AVX2(void); +BoolInt CPU_IsSupported_SSSE3(void); +BoolInt CPU_IsSupported_SSE41(void); +BoolInt CPU_IsSupported_SHA(void); +BoolInt CPU_IsSupported_PageGB(void); + +#elif defined(MY_CPU_ARM_OR_ARM64) + +BoolInt CPU_IsSupported_CRC32(void); +BoolInt CPU_IsSupported_NEON(void); + +#if defined(_WIN32) +BoolInt CPU_IsSupported_CRYPTO(void); +#define CPU_IsSupported_SHA1 CPU_IsSupported_CRYPTO +#define CPU_IsSupported_SHA2 CPU_IsSupported_CRYPTO +#define CPU_IsSupported_AES CPU_IsSupported_CRYPTO +#else +BoolInt CPU_IsSupported_SHA1(void); +BoolInt CPU_IsSupported_SHA2(void); +BoolInt CPU_IsSupported_AES(void); +#endif + +#endif + +#if defined(__APPLE__) +int My_sysctlbyname_Get(const char *name, void *buf, size_t *bufSize); +int My_sysctlbyname_Get_UInt32(const char *name, UInt32 *val); #endif EXTERN_C_END diff --git a/common/LZMA/SDK/C/LzFind.c b/common/LZMA/SDK/C/LzFind.c index 1c8ee6b..1b73c28 100644 --- a/common/LZMA/SDK/C/LzFind.c +++ b/common/LZMA/SDK/C/LzFind.c @@ -1,754 +1,1628 @@ /* LzFind.c -- Match finder for LZ algorithms -2009-04-22 : Igor Pavlov : Public domain */ +2021-11-29 : Igor Pavlov : Public domain */ + +#include "Precomp.h" #include +// #include +#include "CpuArch.h" #include "LzFind.h" #include "LzHash.h" +#define kBlockMoveAlign (1 << 7) // alignment for memmove() +#define kBlockSizeAlign (1 << 16) // alignment for block allocation +#define kBlockSizeReserveMin (1 << 24) // it's 1/256 from 4 GB dictinary + #define kEmptyHashValue 0 -#define kMaxValForNormalize ((UInt32)0xFFFFFFFF) -#define kNormalizeStepMin (1 << 10) /* it must be power of 2 */ -#define kNormalizeMask (~(kNormalizeStepMin - 1)) -#define kMaxHistorySize ((UInt32)3 << 30) -#define kStartMaxLen 3 +#define kMaxValForNormalize ((UInt32)0) +// #define kMaxValForNormalize ((UInt32)(1 << 20) + 0xFFF) // for debug -static void LzInWindow_Free(CMatchFinder *p, ISzAlloc *alloc) +// #define kNormalizeAlign (1 << 7) // alignment for speculated accesses + +#define GET_AVAIL_BYTES(p) \ + Inline_MatchFinder_GetNumAvailableBytes(p) + + +// #define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) +#define kFix5HashSize kFix4HashSize + +/* + HASH2_CALC: + if (hv) match, then cur[0] and cur[1] also match +*/ +#define HASH2_CALC hv = GetUi16(cur); + +// (crc[0 ... 255] & 0xFF) provides one-to-one correspondence to [0 ... 255] + +/* + HASH3_CALC: + if (cur[0]) and (h2) match, then cur[1] also match + if (cur[0]) and (hv) match, then cur[1] and cur[2] also match +*/ +#define HASH3_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + hv = (temp ^ ((UInt32)cur[2] << 8)) & p->hashMask; } + +#define HASH4_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + temp ^= ((UInt32)cur[2] << 8); \ + h3 = temp & (kHash3Size - 1); \ + hv = (temp ^ (p->crc[cur[3]] << kLzHash_CrcShift_1)) & p->hashMask; } + +#define HASH5_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + temp ^= ((UInt32)cur[2] << 8); \ + h3 = temp & (kHash3Size - 1); \ + temp ^= (p->crc[cur[3]] << kLzHash_CrcShift_1); \ + /* h4 = temp & p->hash4Mask; */ /* (kHash4Size - 1); */ \ + hv = (temp ^ (p->crc[cur[4]] << kLzHash_CrcShift_2)) & p->hashMask; } + +#define HASH_ZIP_CALC hv = ((cur[2] | ((UInt32)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF; + + +static void LzInWindow_Free(CMatchFinder *p, ISzAllocPtr alloc) { - if (!p->directInput) - { - alloc->Free(alloc, p->bufferBase); - p->bufferBase = 0; - } + if (!p->directInput) + { + ISzAlloc_Free(alloc, p->bufferBase); + p->bufferBase = NULL; + } } -/* keepSizeBefore + keepSizeAfter + keepSizeReserv must be < 4G) */ -static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAlloc *alloc) +static int LzInWindow_Create2(CMatchFinder *p, UInt32 blockSize, ISzAllocPtr alloc) { - UInt32 blockSize = p->keepSizeBefore + p->keepSizeAfter + keepSizeReserv; - if (p->directInput) + if (blockSize == 0) + return 0; + if (!p->bufferBase || p->blockSize != blockSize) + { + // size_t blockSizeT; + LzInWindow_Free(p, alloc); + p->blockSize = blockSize; + // blockSizeT = blockSize; + + // printf("\nblockSize = 0x%x\n", blockSize); + /* + #if defined _WIN64 + // we can allocate 4GiB, but still use UInt32 for (p->blockSize) + // we use UInt32 type for (p->blockSize), because + // we don't want to wrap over 4 GiB, + // when we use (p->streamPos - p->pos) that is UInt32. + if (blockSize >= (UInt32)0 - (UInt32)kBlockSizeAlign) { - p->blockSize = blockSize; - return 1; + blockSizeT = ((size_t)1 << 32); + printf("\nchanged to blockSizeT = 4GiB\n"); } - if (p->bufferBase == 0 || p->blockSize != blockSize) - { - LzInWindow_Free(p, alloc); - p->blockSize = blockSize; - p->bufferBase = (Byte *)alloc->Alloc(alloc, (size_t)blockSize); - } - return (p->bufferBase != 0); + #endif + */ + + p->bufferBase = (Byte *)ISzAlloc_Alloc(alloc, blockSize); + // printf("\nbufferBase = %p\n", p->bufferBase); + // return 0; // for debug + } + return (p->bufferBase != NULL); } -Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } -Byte MatchFinder_GetIndexByte(CMatchFinder *p, Int32 index) { return p->buffer[index]; } +static const Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } -UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return p->streamPos - p->pos; } +static UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return GET_AVAIL_BYTES(p); } -void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue) -{ - p->posLimit -= subValue; - p->pos -= subValue; - p->streamPos -= subValue; -} +MY_NO_INLINE static void MatchFinder_ReadBlock(CMatchFinder *p) { - if (p->streamEndWasReached || p->result != SZ_OK) - return; - if (p->directInput) + if (p->streamEndWasReached || p->result != SZ_OK) + return; + + /* We use (p->streamPos - p->pos) value. + (p->streamPos < p->pos) is allowed. */ + + if (p->directInput) + { + UInt32 curSize = 0xFFFFFFFF - GET_AVAIL_BYTES(p); + if (curSize > p->directInputRem) + curSize = (UInt32)p->directInputRem; + p->directInputRem -= curSize; + p->streamPos += curSize; + if (p->directInputRem == 0) + p->streamEndWasReached = 1; + return; + } + + for (;;) + { + Byte *dest = p->buffer + GET_AVAIL_BYTES(p); + size_t size = (size_t)(p->bufferBase + p->blockSize - dest); + if (size == 0) { - UInt32 curSize = 0xFFFFFFFF - p->streamPos; - if (curSize > p->directInputRem) - curSize = (UInt32)p->directInputRem; - p->directInputRem -= curSize; - p->streamPos += curSize; - if (p->directInputRem == 0) - p->streamEndWasReached = 1; - return; + /* we call ReadBlock() after NeedMove() and MoveBlock(). + NeedMove() and MoveBlock() povide more than (keepSizeAfter) + to the end of (blockSize). + So we don't execute this branch in normal code flow. + We can go here, if we will call ReadBlock() before NeedMove(), MoveBlock(). + */ + // p->result = SZ_ERROR_FAIL; // we can show error here + return; } - for (;;) + + // #define kRead 3 + // if (size > kRead) size = kRead; // for debug + + p->result = ISeqInStream_Read(p->stream, dest, &size); + if (p->result != SZ_OK) + return; + if (size == 0) { - Byte *dest = p->buffer + (p->streamPos - p->pos); - size_t size = (p->bufferBase + p->blockSize - dest); - if (size == 0) - return; - p->result = p->stream->Read(p->stream, dest, &size); - if (p->result != SZ_OK) - return; - if (size == 0) - { - p->streamEndWasReached = 1; - return; - } - p->streamPos += (UInt32)size; - if (p->streamPos - p->pos > p->keepSizeAfter) - return; + p->streamEndWasReached = 1; + return; } + p->streamPos += (UInt32)size; + if (GET_AVAIL_BYTES(p) > p->keepSizeAfter) + return; + /* here and in another (p->keepSizeAfter) checks we keep on 1 byte more than was requested by Create() function + (GET_AVAIL_BYTES(p) >= p->keepSizeAfter) - minimal required size */ + } + + // on exit: (p->result != SZ_OK || p->streamEndWasReached || GET_AVAIL_BYTES(p) > p->keepSizeAfter) } + + +MY_NO_INLINE void MatchFinder_MoveBlock(CMatchFinder *p) { - memmove(p->bufferBase, - p->buffer - p->keepSizeBefore, - (size_t)(p->streamPos - p->pos + p->keepSizeBefore)); - p->buffer = p->bufferBase + p->keepSizeBefore; + const size_t offset = (size_t)(p->buffer - p->bufferBase) - p->keepSizeBefore; + const size_t keepBefore = (offset & (kBlockMoveAlign - 1)) + p->keepSizeBefore; + p->buffer = p->bufferBase + keepBefore; + memmove(p->bufferBase, + p->bufferBase + (offset & ~((size_t)kBlockMoveAlign - 1)), + keepBefore + (size_t)GET_AVAIL_BYTES(p)); } +/* We call MoveBlock() before ReadBlock(). + So MoveBlock() can be wasteful operation, if the whole input data + can fit in current block even without calling MoveBlock(). + in important case where (dataSize <= historySize) + condition (p->blockSize > dataSize + p->keepSizeAfter) is met + So there is no MoveBlock() in that case case. +*/ + int MatchFinder_NeedMove(CMatchFinder *p) { - if (p->directInput) - return 0; - /* if (p->streamEndWasReached) return 0; */ - return ((size_t)(p->bufferBase + p->blockSize - p->buffer) <= p->keepSizeAfter); + if (p->directInput) + return 0; + if (p->streamEndWasReached || p->result != SZ_OK) + return 0; + return ((size_t)(p->bufferBase + p->blockSize - p->buffer) <= p->keepSizeAfter); } void MatchFinder_ReadIfRequired(CMatchFinder *p) { - if (p->streamEndWasReached) - return; - if (p->keepSizeAfter >= p->streamPos - p->pos) - MatchFinder_ReadBlock(p); -} - -static void MatchFinder_CheckAndMoveAndRead(CMatchFinder *p) -{ - if (MatchFinder_NeedMove(p)) - MatchFinder_MoveBlock(p); + if (p->keepSizeAfter >= GET_AVAIL_BYTES(p)) MatchFinder_ReadBlock(p); } + + static void MatchFinder_SetDefaultSettings(CMatchFinder *p) { - p->cutValue = 32; - p->btMode = 1; - p->numHashBytes = 4; - p->bigHash = 0; + p->cutValue = 32; + p->btMode = 1; + p->numHashBytes = 4; + p->bigHash = 0; } #define kCrcPoly 0xEDB88320 void MatchFinder_Construct(CMatchFinder *p) { - UInt32 i; - p->bufferBase = 0; - p->directInput = 0; - p->hash = 0; - MatchFinder_SetDefaultSettings(p); + unsigned i; + p->bufferBase = NULL; + p->directInput = 0; + p->hash = NULL; + p->expectedDataSize = (UInt64)(Int64)-1; + MatchFinder_SetDefaultSettings(p); - for (i = 0; i < 256; i++) + for (i = 0; i < 256; i++) + { + UInt32 r = (UInt32)i; + unsigned j; + for (j = 0; j < 8; j++) + r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); + p->crc[i] = r; + } +} + +static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->hash); + p->hash = NULL; +} + +void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc) +{ + MatchFinder_FreeThisClassMemory(p, alloc); + LzInWindow_Free(p, alloc); +} + +static CLzRef* AllocRefs(size_t num, ISzAllocPtr alloc) +{ + size_t sizeInBytes = (size_t)num * sizeof(CLzRef); + if (sizeInBytes / sizeof(CLzRef) != num) + return NULL; + return (CLzRef *)ISzAlloc_Alloc(alloc, sizeInBytes); +} + +#if (kBlockSizeReserveMin < kBlockSizeAlign * 2) + #error Stop_Compiling_Bad_Reserve +#endif + + + +static UInt32 GetBlockSize(CMatchFinder *p, UInt32 historySize) +{ + UInt32 blockSize = (p->keepSizeBefore + p->keepSizeAfter); + /* + if (historySize > kMaxHistorySize) + return 0; + */ + // printf("\nhistorySize == 0x%x\n", historySize); + + if (p->keepSizeBefore < historySize || blockSize < p->keepSizeBefore) // if 32-bit overflow + return 0; + + { + const UInt32 kBlockSizeMax = (UInt32)0 - (UInt32)kBlockSizeAlign; + const UInt32 rem = kBlockSizeMax - blockSize; + const UInt32 reserve = (blockSize >> (blockSize < ((UInt32)1 << 30) ? 1 : 2)) + + (1 << 12) + kBlockMoveAlign + kBlockSizeAlign; // do not overflow 32-bit here + if (blockSize >= kBlockSizeMax + || rem < kBlockSizeReserveMin) // we reject settings that will be slow + return 0; + if (reserve >= rem) + blockSize = kBlockSizeMax; + else { - UInt32 r = i; - int j; - for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1)); - p->crc[i] = r; + blockSize += reserve; + blockSize &= ~(UInt32)(kBlockSizeAlign - 1); } + } + // printf("\n LzFind_blockSize = %x\n", blockSize); + // printf("\n LzFind_blockSize = %d\n", blockSize >> 20); + return blockSize; } -static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAlloc *alloc) -{ - alloc->Free(alloc, p->hash); - p->hash = 0; -} - -void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc) -{ - MatchFinder_FreeThisClassMemory(p, alloc); - LzInWindow_Free(p, alloc); -} - -static CLzRef* AllocRefs(UInt32 num, ISzAlloc *alloc) -{ - size_t sizeInBytes = (size_t)num * sizeof(CLzRef); - if (sizeInBytes / sizeof(CLzRef) != num) - return 0; - return (CLzRef *)alloc->Alloc(alloc, sizeInBytes); -} int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAlloc *alloc) + ISzAllocPtr alloc) { - UInt32 sizeReserv; - if (historySize > kMaxHistorySize) - { - MatchFinder_Free(p, alloc); - return 0; - } - sizeReserv = historySize >> 1; - if (historySize > ((UInt32)2 << 30)) - sizeReserv = historySize >> 2; - sizeReserv += (keepAddBufferBefore + matchMaxLen + keepAddBufferAfter) / 2 + (1 << 19); + /* we need one additional byte in (p->keepSizeBefore), + since we use MoveBlock() after (p->pos++) and before dictionary using */ + // keepAddBufferBefore = (UInt32)0xFFFFFFFF - (1 << 22); // for debug + p->keepSizeBefore = historySize + keepAddBufferBefore + 1; - p->keepSizeBefore = historySize + keepAddBufferBefore + 1; - p->keepSizeAfter = matchMaxLen + keepAddBufferAfter; - /* we need one additional byte, since we use MoveBlock after pos++ and before dictionary using */ - if (LzInWindow_Create(p, sizeReserv, alloc)) + keepAddBufferAfter += matchMaxLen; + /* we need (p->keepSizeAfter >= p->numHashBytes) */ + if (keepAddBufferAfter < p->numHashBytes) + keepAddBufferAfter = p->numHashBytes; + // keepAddBufferAfter -= 2; // for debug + p->keepSizeAfter = keepAddBufferAfter; + + if (p->directInput) + p->blockSize = 0; + if (p->directInput || LzInWindow_Create2(p, GetBlockSize(p, historySize), alloc)) + { + const UInt32 newCyclicBufferSize = historySize + 1; // do not change it + UInt32 hs; + p->matchMaxLen = matchMaxLen; { - UInt32 newCyclicBufferSize = historySize + 1; - UInt32 hs; - p->matchMaxLen = matchMaxLen; + // UInt32 hs4; + p->fixedHashSize = 0; + hs = (1 << 16) - 1; + if (p->numHashBytes != 2) + { + hs = historySize; + if (hs > p->expectedDataSize) + hs = (UInt32)p->expectedDataSize; + if (hs != 0) + hs--; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + // we propagated 16 bits in (hs). Low 16 bits must be set later + hs >>= 1; + if (hs >= (1 << 24)) { - p->fixedHashSize = 0; - if (p->numHashBytes == 2) - hs = (1 << 16) - 1; - else - { - hs = historySize - 1; - hs |= (hs >> 1); - hs |= (hs >> 2); - hs |= (hs >> 4); - hs |= (hs >> 8); - hs >>= 1; - hs |= 0xFFFF; /* don't change it! It's required for Deflate */ - if (hs > (1 << 24)) - { - if (p->numHashBytes == 3) - hs = (1 << 24) - 1; - else - hs >>= 1; - } - } - p->hashMask = hs; - hs++; - if (p->numHashBytes > 2) p->fixedHashSize += kHash2Size; - if (p->numHashBytes > 3) p->fixedHashSize += kHash3Size; - if (p->numHashBytes > 4) p->fixedHashSize += kHash4Size; - hs += p->fixedHashSize; + if (p->numHashBytes == 3) + hs = (1 << 24) - 1; + else + hs >>= 1; + /* if (bigHash) mode, GetHeads4b() in LzFindMt.c needs (hs >= ((1 << 24) - 1))) */ } + + // hs = ((UInt32)1 << 25) - 1; // for test + + // (hash_size >= (1 << 16)) : Required for (numHashBytes > 2) + hs |= (1 << 16) - 1; /* don't change it! */ + + // bt5: we adjust the size with recommended minimum size + if (p->numHashBytes >= 5) + hs |= (256 << kLzHash_CrcShift_2) - 1; + } + p->hashMask = hs; + hs++; + + /* + hs4 = (1 << 20); + if (hs4 > hs) + hs4 = hs; + // hs4 = (1 << 16); // for test + p->hash4Mask = hs4 - 1; + */ + + if (p->numHashBytes > 2) p->fixedHashSize += kHash2Size; + if (p->numHashBytes > 3) p->fixedHashSize += kHash3Size; + // if (p->numHashBytes > 4) p->fixedHashSize += hs4; // kHash4Size; + hs += p->fixedHashSize; + } { - UInt32 prevSize = p->hashSizeSum + p->numSons; - UInt32 newSize; - p->historySize = historySize; - p->hashSizeSum = hs; - p->cyclicBufferSize = newCyclicBufferSize; - p->numSons = (p->btMode ? newCyclicBufferSize * 2 : newCyclicBufferSize); - newSize = p->hashSizeSum + p->numSons; - if (p->hash != 0 && prevSize == newSize) - return 1; - MatchFinder_FreeThisClassMemory(p, alloc); - p->hash = AllocRefs(newSize, alloc); - if (p->hash != 0) - { - p->son = p->hash + p->hashSizeSum; - return 1; - } + size_t newSize; + size_t numSons; + p->historySize = historySize; + p->hashSizeSum = hs; + p->cyclicBufferSize = newCyclicBufferSize; // it must be = (historySize + 1) + + numSons = newCyclicBufferSize; + if (p->btMode) + numSons <<= 1; + newSize = hs + numSons; + + // aligned size is not required here, but it can be better for some loops + #define NUM_REFS_ALIGN_MASK 0xF + newSize = (newSize + NUM_REFS_ALIGN_MASK) & ~(size_t)NUM_REFS_ALIGN_MASK; + + if (p->hash && p->numRefs == newSize) + return 1; + + MatchFinder_FreeThisClassMemory(p, alloc); + p->numRefs = newSize; + p->hash = AllocRefs(newSize, alloc); + + if (p->hash) + { + p->son = p->hash + p->hashSizeSum; + return 1; + } } - } - MatchFinder_Free(p, alloc); - return 0; + } + + MatchFinder_Free(p, alloc); + return 0; } + static void MatchFinder_SetLimits(CMatchFinder *p) { - UInt32 limit = kMaxValForNormalize - p->pos; - UInt32 limit2 = p->cyclicBufferSize - p->cyclicBufferPos; - if (limit2 < limit) - limit = limit2; - limit2 = p->streamPos - p->pos; - if (limit2 <= p->keepSizeAfter) + UInt32 k; + UInt32 n = kMaxValForNormalize - p->pos; + if (n == 0) + n = (UInt32)(Int32)-1; // we allow (pos == 0) at start even with (kMaxValForNormalize == 0) + + k = p->cyclicBufferSize - p->cyclicBufferPos; + if (k < n) + n = k; + + k = GET_AVAIL_BYTES(p); + { + const UInt32 ksa = p->keepSizeAfter; + UInt32 mm = p->matchMaxLen; + if (k > ksa) + k -= ksa; // we must limit exactly to keepSizeAfter for ReadBlock + else if (k >= mm) { - if (limit2 > 0) - limit2 = 1; + // the limitation for (p->lenLimit) update + k -= mm; // optimization : to reduce the number of checks + k++; + // k = 1; // non-optimized version : for debug } else - limit2 -= p->keepSizeAfter; - if (limit2 < limit) - limit = limit2; { - UInt32 lenLimit = p->streamPos - p->pos; - if (lenLimit > p->matchMaxLen) - lenLimit = p->matchMaxLen; - p->lenLimit = lenLimit; + mm = k; + if (k != 0) + k = 1; } - p->posLimit = p->pos + limit; + p->lenLimit = mm; + } + if (k < n) + n = k; + + p->posLimit = p->pos + n; } + +void MatchFinder_Init_LowHash(CMatchFinder *p) +{ + size_t i; + CLzRef *items = p->hash; + const size_t numItems = p->fixedHashSize; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_HighHash(CMatchFinder *p) +{ + size_t i; + CLzRef *items = p->hash + p->fixedHashSize; + const size_t numItems = (size_t)p->hashMask + 1; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_4(CMatchFinder *p) +{ + p->buffer = p->bufferBase; + { + /* kEmptyHashValue = 0 (Zero) is used in hash tables as NO-VALUE marker. + the code in CMatchFinderMt expects (pos = 1) */ + p->pos = + p->streamPos = + 1; // it's smallest optimal value. do not change it + // 0; // for debug + } + p->result = SZ_OK; + p->streamEndWasReached = 0; +} + + +// (CYC_TO_POS_OFFSET == 0) is expected by some optimized code +#define CYC_TO_POS_OFFSET 0 +// #define CYC_TO_POS_OFFSET 1 // for debug + void MatchFinder_Init(CMatchFinder *p) { - UInt32 i; - for (i = 0; i < p->hashSizeSum; i++) - p->hash[i] = kEmptyHashValue; - p->cyclicBufferPos = 0; - p->buffer = p->bufferBase; - p->pos = p->streamPos = p->cyclicBufferSize; - p->result = SZ_OK; - p->streamEndWasReached = 0; - MatchFinder_ReadBlock(p); - MatchFinder_SetLimits(p); + MatchFinder_Init_HighHash(p); + MatchFinder_Init_LowHash(p); + MatchFinder_Init_4(p); + // if (readData) + MatchFinder_ReadBlock(p); + + /* if we init (cyclicBufferPos = pos), then we can use one variable + instead of both (cyclicBufferPos) and (pos) : only before (cyclicBufferPos) wrapping */ + p->cyclicBufferPos = (p->pos - CYC_TO_POS_OFFSET); // init with relation to (pos) + // p->cyclicBufferPos = 0; // smallest value + // p->son[0] = p->son[1] = 0; // unused: we can init skipped record for speculated accesses. + MatchFinder_SetLimits(p); } -static UInt32 MatchFinder_GetSubValue(CMatchFinder *p) + + +#ifdef MY_CPU_X86_OR_AMD64 + #if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 8) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1900) + #define USE_SATUR_SUB_128 + #define USE_AVX2 + #define ATTRIB_SSE41 __attribute__((__target__("sse4.1"))) + #define ATTRIB_AVX2 __attribute__((__target__("avx2"))) + #elif defined(_MSC_VER) + #if (_MSC_VER >= 1600) + #define USE_SATUR_SUB_128 + #if (_MSC_VER >= 1900) + #define USE_AVX2 + #include // avx + #endif + #endif + #endif + +// #elif defined(MY_CPU_ARM_OR_ARM64) +#elif defined(MY_CPU_ARM64) + + #if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 8) + #define USE_SATUR_SUB_128 + #ifdef MY_CPU_ARM64 + // #define ATTRIB_SSE41 __attribute__((__target__(""))) + #else + // #define ATTRIB_SSE41 __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) + #endif + + #elif defined(_MSC_VER) + #if (_MSC_VER >= 1910) + #define USE_SATUR_SUB_128 + #endif + #endif + + #if defined(_MSC_VER) && defined(MY_CPU_ARM64) + #include + #else + #include + #endif + +#endif + +/* +#ifndef ATTRIB_SSE41 + #define ATTRIB_SSE41 +#endif +#ifndef ATTRIB_AVX2 + #define ATTRIB_AVX2 +#endif +*/ + +#ifdef USE_SATUR_SUB_128 + +// #define _SHOW_HW_STATUS + +#ifdef _SHOW_HW_STATUS +#include +#define _PRF(x) x +_PRF(;) +#else +#define _PRF(x) +#endif + +#ifdef MY_CPU_ARM_OR_ARM64 + +#ifdef MY_CPU_ARM64 +// #define FORCE_SATUR_SUB_128 +#endif + +typedef uint32x4_t v128; +#define SASUB_128(i) \ + *(v128 *)(void *)(items + (i) * 4) = \ + vsubq_u32(vmaxq_u32(*(const v128 *)(const void *)(items + (i) * 4), sub2), sub2); + +#else + +#include // sse4.1 + +typedef __m128i v128; +#define SASUB_128(i) \ + *(v128 *)(void *)(items + (i) * 4) = \ + _mm_sub_epi32(_mm_max_epu32(*(const v128 *)(const void *)(items + (i) * 4), sub2), sub2); // SSE 4.1 + +#endif + + + +MY_NO_INLINE +static +#ifdef ATTRIB_SSE41 +ATTRIB_SSE41 +#endif +void +MY_FAST_CALL +LzFind_SaturSub_128(UInt32 subValue, CLzRef *items, const CLzRef *lim) { - return (p->pos - p->historySize - 1) & kNormalizeMask; + v128 sub2 = + #ifdef MY_CPU_ARM_OR_ARM64 + vdupq_n_u32(subValue); + #else + _mm_set_epi32((Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue); + #endif + do + { + SASUB_128(0) + SASUB_128(1) + SASUB_128(2) + SASUB_128(3) + items += 4 * 4; + } + while (items != lim); } -void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, UInt32 numItems) + + +#ifdef USE_AVX2 + +#include // avx + +#define SASUB_256(i) *(__m256i *)(void *)(items + (i) * 8) = _mm256_sub_epi32(_mm256_max_epu32(*(const __m256i *)(const void *)(items + (i) * 8), sub2), sub2); // AVX2 + +MY_NO_INLINE +static +#ifdef ATTRIB_AVX2 +ATTRIB_AVX2 +#endif +void +MY_FAST_CALL +LzFind_SaturSub_256(UInt32 subValue, CLzRef *items, const CLzRef *lim) { - UInt32 i; - for (i = 0; i < numItems; i++) + __m256i sub2 = _mm256_set_epi32( + (Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue, + (Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue); + do + { + SASUB_256(0) + SASUB_256(1) + items += 2 * 8; + } + while (items != lim); +} +#endif // USE_AVX2 + +#ifndef FORCE_SATUR_SUB_128 +typedef void (MY_FAST_CALL *LZFIND_SATUR_SUB_CODE_FUNC)( + UInt32 subValue, CLzRef *items, const CLzRef *lim); +static LZFIND_SATUR_SUB_CODE_FUNC g_LzFind_SaturSub; +#endif // FORCE_SATUR_SUB_128 + +#endif // USE_SATUR_SUB_128 + + +// kEmptyHashValue must be zero +// #define SASUB_32(i) v = items[i]; m = v - subValue; if (v < subValue) m = kEmptyHashValue; items[i] = m; +#define SASUB_32(i) v = items[i]; if (v < subValue) v = subValue; items[i] = v - subValue; + +#ifdef FORCE_SATUR_SUB_128 + +#define DEFAULT_SaturSub LzFind_SaturSub_128 + +#else + +#define DEFAULT_SaturSub LzFind_SaturSub_32 + +MY_NO_INLINE +static +void +MY_FAST_CALL +LzFind_SaturSub_32(UInt32 subValue, CLzRef *items, const CLzRef *lim) +{ + do + { + UInt32 v; + SASUB_32(0) + SASUB_32(1) + SASUB_32(2) + SASUB_32(3) + SASUB_32(4) + SASUB_32(5) + SASUB_32(6) + SASUB_32(7) + items += 8; + } + while (items != lim); +} + +#endif + + +MY_NO_INLINE +void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems) +{ + #define K_NORM_ALIGN_BLOCK_SIZE (1 << 6) + + CLzRef *lim; + + for (; numItems != 0 && ((unsigned)(ptrdiff_t)items & (K_NORM_ALIGN_BLOCK_SIZE - 1)) != 0; numItems--) + { + UInt32 v; + SASUB_32(0); + items++; + } + + { + #define K_NORM_ALIGN_MASK (K_NORM_ALIGN_BLOCK_SIZE / 4 - 1) + lim = items + (numItems & ~(size_t)K_NORM_ALIGN_MASK); + numItems &= K_NORM_ALIGN_MASK; + if (items != lim) { - UInt32 value = items[i]; - if (value <= subValue) - value = kEmptyHashValue; + #if defined(USE_SATUR_SUB_128) && !defined(FORCE_SATUR_SUB_128) + if (g_LzFind_SaturSub) + g_LzFind_SaturSub(subValue, items, lim); else - value -= subValue; - items[i] = value; + #endif + DEFAULT_SaturSub(subValue, items, lim); } + items = lim; + } + + + for (; numItems != 0; numItems--) + { + UInt32 v; + SASUB_32(0); + items++; + } } -static void MatchFinder_Normalize(CMatchFinder *p) -{ - UInt32 subValue = MatchFinder_GetSubValue(p); - MatchFinder_Normalize3(subValue, p->hash, p->hashSizeSum + p->numSons); - MatchFinder_ReduceOffsets(p, subValue); -} + +// call MatchFinder_CheckLimits() only after (p->pos++) update + +MY_NO_INLINE static void MatchFinder_CheckLimits(CMatchFinder *p) { - if (p->pos == kMaxValForNormalize) - MatchFinder_Normalize(p); - if (!p->streamEndWasReached && p->keepSizeAfter == p->streamPos - p->pos) - MatchFinder_CheckAndMoveAndRead(p); - if (p->cyclicBufferPos == p->cyclicBufferSize) - p->cyclicBufferPos = 0; - MatchFinder_SetLimits(p); + if (// !p->streamEndWasReached && p->result == SZ_OK && + p->keepSizeAfter == GET_AVAIL_BYTES(p)) + { + // we try to read only in exact state (p->keepSizeAfter == GET_AVAIL_BYTES(p)) + if (MatchFinder_NeedMove(p)) + MatchFinder_MoveBlock(p); + MatchFinder_ReadBlock(p); + } + + if (p->pos == kMaxValForNormalize) + if (GET_AVAIL_BYTES(p) >= p->numHashBytes) // optional optimization for last bytes of data. + /* + if we disable normalization for last bytes of data, and + if (data_size == 4 GiB), we don't call wastfull normalization, + but (pos) will be wrapped over Zero (0) in that case. + And we cannot resume later to normal operation + */ + { + // MatchFinder_Normalize(p); + /* after normalization we need (p->pos >= p->historySize + 1); */ + /* we can reduce subValue to aligned value, if want to keep alignment + of (p->pos) and (p->buffer) for speculated accesses. */ + const UInt32 subValue = (p->pos - p->historySize - 1) /* & ~(UInt32)(kNormalizeAlign - 1) */; + // const UInt32 subValue = (1 << 15); // for debug + // printf("\nMatchFinder_Normalize() subValue == 0x%x\n", subValue); + size_t numSonRefs = p->cyclicBufferSize; + if (p->btMode) + numSonRefs <<= 1; + Inline_MatchFinder_ReduceOffsets(p, subValue); + MatchFinder_Normalize3(subValue, p->hash, (size_t)p->hashSizeSum + numSonRefs); + } + + if (p->cyclicBufferPos == p->cyclicBufferSize) + p->cyclicBufferPos = 0; + + MatchFinder_SetLimits(p); } -static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) + +/* + (lenLimit > maxLen) +*/ +MY_FORCE_INLINE +static UInt32 * Hc_GetMatchesSpec(size_t lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, + UInt32 *d, unsigned maxLen) { - son[_cyclicBufferPos] = curMatch; - for (;;) + /* + son[_cyclicBufferPos] = curMatch; + for (;;) + { + UInt32 delta = pos - curMatch; + if (cutValue-- == 0 || delta >= _cyclicBufferSize) + return d; { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) - return distances; + const Byte *pb = cur - delta; + curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; + if (pb[maxLen] == cur[maxLen] && *pb == *cur) + { + UInt32 len = 0; + while (++len != lenLimit) + if (pb[len] != cur[len]) + break; + if (maxLen < len) { - const Byte *pb = cur - delta; - curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; - if (pb[maxLen] == cur[maxLen] && *pb == *cur) - { - UInt32 len = 0; - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - if (maxLen < len) - { - *distances++ = maxLen = len; - *distances++ = delta - 1; - if (len == lenLimit) - return distances; - } - } + maxLen = len; + *d++ = len; + *d++ = delta - 1; + if (len == lenLimit) + return d; } + } } + } + */ + + const Byte *lim = cur + lenLimit; + son[_cyclicBufferPos] = curMatch; + + do + { + UInt32 delta; + + if (curMatch == 0) + break; + // if (curMatch2 >= curMatch) return NULL; + delta = pos - curMatch; + if (delta >= _cyclicBufferSize) + break; + { + ptrdiff_t diff; + curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; + diff = (ptrdiff_t)0 - (ptrdiff_t)delta; + if (cur[maxLen] == cur[(ptrdiff_t)maxLen + diff]) + { + const Byte *c = cur; + while (*c == c[diff]) + { + if (++c == lim) + { + d[0] = (UInt32)(lim - cur); + d[1] = delta - 1; + return d + 2; + } + } + { + const unsigned len = (unsigned)(c - cur); + if (maxLen < len) + { + maxLen = len; + d[0] = (UInt32)len; + d[1] = delta - 1; + d += 2; + } + } + } + } + } + while (--cutValue); + + return d; } + +MY_FORCE_INLINE UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, + UInt32 *d, UInt32 maxLen) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; - for (;;) + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; + + UInt32 cmCheck; + + // if (curMatch >= pos) { *ptr0 = *ptr1 = kEmptyHashValue; return NULL; } + + cmCheck = (UInt32)(pos - _cyclicBufferSize); + if ((UInt32)pos <= _cyclicBufferSize) + cmCheck = 0; + + if (cmCheck < curMatch) + do + { + const UInt32 delta = pos - curMatch; { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + const Byte *pb = cur - delta; + unsigned len = (len0 < len1 ? len0 : len1); + const UInt32 pair0 = pair[0]; + if (pb[len] == cur[len]) + { + if (++len != lenLimit && pb[len] == cur[len]) + while (++len != lenLimit) + if (pb[len] != cur[len]) + break; + if (maxLen < len) { - *ptr0 = *ptr1 = kEmptyHashValue; - return distances; - } - { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); - const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); - if (pb[len] == cur[len]) - { - if (++len != lenLimit && pb[len] == cur[len]) - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - if (maxLen < len) - { - *distances++ = maxLen = len; - *distances++ = delta - 1; - if (len == lenLimit) - { - *ptr1 = pair[0]; - *ptr0 = pair[1]; - return distances; - } - } - } - if (pb[len] < cur[len]) - { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; + maxLen = (UInt32)len; + *d++ = (UInt32)len; + *d++ = delta - 1; + if (len == lenLimit) + { + *ptr1 = pair0; + *ptr0 = pair[1]; + return d; + } } + } + if (pb[len] < cur[len]) + { + *ptr1 = curMatch; + // const UInt32 curMatch2 = pair[1]; + // if (curMatch2 >= curMatch) { *ptr0 = *ptr1 = kEmptyHashValue; return NULL; } + // curMatch = curMatch2; + curMatch = pair[1]; + ptr1 = pair + 1; + len1 = len; + } + else + { + *ptr0 = curMatch; + curMatch = pair[0]; + ptr0 = pair; + len0 = len; + } } - } + } + while(--cutValue && cmCheck < curMatch); + + *ptr0 = *ptr1 = kEmptyHashValue; + return d; } + static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; - for (;;) + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; + + UInt32 cmCheck; + + cmCheck = (UInt32)(pos - _cyclicBufferSize); + if ((UInt32)pos <= _cyclicBufferSize) + cmCheck = 0; + + if (// curMatch >= pos || // failure + cmCheck < curMatch) + do + { + const UInt32 delta = pos - curMatch; { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + const Byte *pb = cur - delta; + unsigned len = (len0 < len1 ? len0 : len1); + if (pb[len] == cur[len]) + { + while (++len != lenLimit) + if (pb[len] != cur[len]) + break; { - *ptr0 = *ptr1 = kEmptyHashValue; + if (len == lenLimit) + { + *ptr1 = pair[0]; + *ptr0 = pair[1]; return; + } } - { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); - const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); - if (pb[len] == cur[len]) - { - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - { - if (len == lenLimit) - { - *ptr1 = pair[0]; - *ptr0 = pair[1]; - return; - } - } - } - if (pb[len] < cur[len]) - { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; - } - } + } + if (pb[len] < cur[len]) + { + *ptr1 = curMatch; + curMatch = pair[1]; + ptr1 = pair + 1; + len1 = len; + } + else + { + *ptr0 = curMatch; + curMatch = pair[0]; + ptr0 = pair; + len0 = len; + } } + } + while(--cutValue && cmCheck < curMatch); + + *ptr0 = *ptr1 = kEmptyHashValue; + return; } + #define MOVE_POS \ ++p->cyclicBufferPos; \ p->buffer++; \ - if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p); + { const UInt32 pos1 = p->pos + 1; p->pos = pos1; if (pos1 == p->posLimit) MatchFinder_CheckLimits(p); } -#define MOVE_POS_RET MOVE_POS return offset; +#define MOVE_POS_RET MOVE_POS return distances; -static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } +MY_NO_INLINE +static void MatchFinder_MovePos(CMatchFinder *p) +{ + /* we go here at the end of stream data, when (avail < num_hash_bytes) + We don't update sons[cyclicBufferPos << btMode]. + So (sons) record will contain junk. And we cannot resume match searching + to normal operation, even if we will provide more input data in buffer. + p->sons[p->cyclicBufferPos << p->btMode] = 0; // kEmptyHashValue + if (p->btMode) + p->sons[(p->cyclicBufferPos << p->btMode) + 1] = 0; // kEmptyHashValue + */ + MOVE_POS; +} #define GET_MATCHES_HEADER2(minLen, ret_op) \ - UInt32 lenLimit; UInt32 hashValue; const Byte *cur; UInt32 curMatch; \ - lenLimit = p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ + unsigned lenLimit; UInt32 hv; Byte *cur; UInt32 curMatch; \ + lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ cur = p->buffer; -#define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0) -#define SKIP_HEADER(minLen) GET_MATCHES_HEADER2(minLen, continue) +#define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return distances) +#define SKIP_HEADER(minLen) do { GET_MATCHES_HEADER2(minLen, continue) -#define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue +#define MF_PARAMS(p) lenLimit, curMatch, p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue -#define GET_MATCHES_FOOTER(offset, maxLen) \ - offset = (UInt32)(GetMatchesSpec1(lenLimit, curMatch, MF_PARAMS(p), \ - distances + offset, maxLen) - distances); MOVE_POS_RET; +#define SKIP_FOOTER SkipMatchesSpec(MF_PARAMS(p)); MOVE_POS; } while (--num); -#define SKIP_FOOTER \ - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; +#define GET_MATCHES_FOOTER_BASE(_maxLen_, func) \ + distances = func(MF_PARAMS(p), \ + distances, (UInt32)_maxLen_); MOVE_POS_RET; -static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +#define GET_MATCHES_FOOTER_BT(_maxLen_) \ + GET_MATCHES_FOOTER_BASE(_maxLen_, GetMatchesSpec1) + +#define GET_MATCHES_FOOTER_HC(_maxLen_) \ + GET_MATCHES_FOOTER_BASE(_maxLen_, Hc_GetMatchesSpec) + + + +#define UPDATE_maxLen { \ + const ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)d2; \ + const Byte *c = cur + maxLen; \ + const Byte *lim = cur + lenLimit; \ + for (; c != lim; c++) if (*(c + diff) != *c) break; \ + maxLen = (unsigned)(c - cur); } + +static UInt32* Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; - GET_MATCHES_HEADER(2) - HASH2_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - offset = 0; - GET_MATCHES_FOOTER(offset, 1) + GET_MATCHES_HEADER(2) + HASH2_CALC; + curMatch = p->hash[hv]; + p->hash[hv] = p->pos; + GET_MATCHES_FOOTER_BT(1) } -UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +UInt32* Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; - GET_MATCHES_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - offset = 0; - GET_MATCHES_FOOTER(offset, 2) + GET_MATCHES_HEADER(3) + HASH_ZIP_CALC; + curMatch = p->hash[hv]; + p->hash[hv] = p->pos; + GET_MATCHES_FOOTER_BT(2) } -static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) + +#define SET_mmm \ + mmm = p->cyclicBufferSize; \ + if (pos < mmm) \ + mmm = pos; + + +static UInt32* Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 hash2Value, delta2, maxLen, offset; - GET_MATCHES_HEADER(3) + UInt32 mmm; + UInt32 h2, d2, pos; + unsigned maxLen; + UInt32 *hash; + GET_MATCHES_HEADER(3) - HASH3_CALC; + HASH3_CALC; - delta2 = p->pos - p->hash[hash2Value]; - curMatch = p->hash[kFix3HashSize + hashValue]; + hash = p->hash; + pos = p->pos; - p->hash[hash2Value] = - p->hash[kFix3HashSize + hashValue] = p->pos; + d2 = pos - hash[h2]; - maxLen = 2; - offset = 0; - if (delta2 < p->cyclicBufferSize && *(cur - delta2) == *cur) + curMatch = (hash + kFix3HashSize)[hv]; + + hash[h2] = pos; + (hash + kFix3HashSize)[hv] = pos; + + SET_mmm + + maxLen = 2; + + if (d2 < mmm && *(cur - d2) == *cur) + { + UPDATE_maxLen + distances[0] = (UInt32)maxLen; + distances[1] = d2 - 1; + distances += 2; + if (maxLen == lenLimit) { - for (; maxLen != lenLimit; maxLen++) - if (cur[(ptrdiff_t)maxLen - delta2] != cur[maxLen]) - break; - distances[0] = maxLen; - distances[1] = delta2 - 1; - offset = 2; - if (maxLen == lenLimit) - { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); - MOVE_POS_RET; - } + SkipMatchesSpec(MF_PARAMS(p)); + MOVE_POS_RET; } - GET_MATCHES_FOOTER(offset, maxLen) + } + + GET_MATCHES_FOOTER_BT(maxLen) } -static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) + +static UInt32* Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 hash2Value, hash3Value, delta2, delta3, maxLen, offset; - GET_MATCHES_HEADER(4) + UInt32 mmm; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen; + UInt32 *hash; + GET_MATCHES_HEADER(4) - HASH4_CALC; + HASH4_CALC; - delta2 = p->pos - p->hash[hash2Value]; - delta3 = p->pos - p->hash[kFix3HashSize + hash3Value]; - curMatch = p->hash[kFix4HashSize + hashValue]; + hash = p->hash; + pos = p->pos; - p->hash[hash2Value] = - p->hash[kFix3HashSize + hash3Value] = - p->hash[kFix4HashSize + hashValue] = p->pos; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + curMatch = (hash + kFix4HashSize)[hv]; - maxLen = 1; - offset = 0; - if (delta2 < p->cyclicBufferSize && *(cur - delta2) == *cur) + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[hv] = pos; + + SET_mmm + + maxLen = 3; + + for (;;) + { + if (d2 < mmm && *(cur - d2) == *cur) { - distances[0] = maxLen = 2; - distances[1] = delta2 - 1; - offset = 2; + distances[0] = 2; + distances[1] = d2 - 1; + distances += 2; + if (*(cur - d2 + 2) == cur[2]) + { + // distances[-2] = 3; + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + d2 = d3; + distances[1] = d3 - 1; + distances += 2; + } + else + break; } - if (delta2 != delta3 && delta3 < p->cyclicBufferSize && *(cur - delta3) == *cur) + else if (d3 < mmm && *(cur - d3) == *cur) { - maxLen = 3; - distances[offset + 1] = delta3 - 1; - offset += 2; - delta2 = delta3; + d2 = d3; + distances[1] = d3 - 1; + distances += 2; } - if (offset != 0) + else + break; + + UPDATE_maxLen + distances[-2] = (UInt32)maxLen; + if (maxLen == lenLimit) { - for (; maxLen != lenLimit; maxLen++) - if (cur[(ptrdiff_t)maxLen - delta2] != cur[maxLen]) - break; - distances[offset - 2] = maxLen; - if (maxLen == lenLimit) - { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); - MOVE_POS_RET; - } + SkipMatchesSpec(MF_PARAMS(p)); + MOVE_POS_RET } - if (maxLen < 3) - maxLen = 3; - GET_MATCHES_FOOTER(offset, maxLen) + break; + } + + GET_MATCHES_FOOTER_BT(maxLen) } -static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) + +static UInt32* Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 hash2Value, hash3Value, delta2, delta3, maxLen, offset; - GET_MATCHES_HEADER(4) + UInt32 mmm; + UInt32 h2, h3, d2, d3, maxLen, pos; + UInt32 *hash; + GET_MATCHES_HEADER(5) - HASH4_CALC; + HASH5_CALC; - delta2 = p->pos - p->hash[hash2Value]; - delta3 = p->pos - p->hash[kFix3HashSize + hash3Value]; - curMatch = p->hash[kFix4HashSize + hashValue]; + hash = p->hash; + pos = p->pos; - p->hash[hash2Value] = - p->hash[kFix3HashSize + hash3Value] = - p->hash[kFix4HashSize + hashValue] = p->pos; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + // d4 = pos - (hash + kFix4HashSize)[h4]; - maxLen = 1; - offset = 0; - if (delta2 < p->cyclicBufferSize && *(cur - delta2) == *cur) + curMatch = (hash + kFix5HashSize)[hv]; + + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + // (hash + kFix4HashSize)[h4] = pos; + (hash + kFix5HashSize)[hv] = pos; + + SET_mmm + + maxLen = 4; + + for (;;) + { + if (d2 < mmm && *(cur - d2) == *cur) { - distances[0] = maxLen = 2; - distances[1] = delta2 - 1; - offset = 2; + distances[0] = 2; + distances[1] = d2 - 1; + distances += 2; + if (*(cur - d2 + 2) == cur[2]) + { + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + distances[1] = d3 - 1; + distances += 2; + d2 = d3; + } + else + break; } - if (delta2 != delta3 && delta3 < p->cyclicBufferSize && *(cur - delta3) == *cur) + else if (d3 < mmm && *(cur - d3) == *cur) { - maxLen = 3; - distances[offset + 1] = delta3 - 1; - offset += 2; - delta2 = delta3; + distances[1] = d3 - 1; + distances += 2; + d2 = d3; } - if (offset != 0) + else + break; + + distances[-2] = 3; + if (*(cur - d2 + 3) != cur[3]) + break; + UPDATE_maxLen + distances[-2] = (UInt32)maxLen; + if (maxLen == lenLimit) { - for (; maxLen != lenLimit; maxLen++) - if (cur[(ptrdiff_t)maxLen - delta2] != cur[maxLen]) - break; - distances[offset - 2] = maxLen; - if (maxLen == lenLimit) - { - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS_RET; - } + SkipMatchesSpec(MF_PARAMS(p)); + MOVE_POS_RET; } - if (maxLen < 3) - maxLen = 3; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances + offset, maxLen) - (distances)); - MOVE_POS_RET + break; + } + + GET_MATCHES_FOOTER_BT(maxLen) } -UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) + +static UInt32* Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; - GET_MATCHES_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances, 2) - (distances)); - MOVE_POS_RET + UInt32 mmm; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen; + UInt32 *hash; + GET_MATCHES_HEADER(4) + + HASH4_CALC; + + hash = p->hash; + pos = p->pos; + + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + curMatch = (hash + kFix4HashSize)[hv]; + + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[hv] = pos; + + SET_mmm + + maxLen = 3; + + for (;;) + { + if (d2 < mmm && *(cur - d2) == *cur) + { + distances[0] = 2; + distances[1] = d2 - 1; + distances += 2; + if (*(cur - d2 + 2) == cur[2]) + { + // distances[-2] = 3; + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + d2 = d3; + distances[1] = d3 - 1; + distances += 2; + } + else + break; + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + d2 = d3; + distances[1] = d3 - 1; + distances += 2; + } + else + break; + + UPDATE_maxLen + distances[-2] = (UInt32)maxLen; + if (maxLen == lenLimit) + { + p->son[p->cyclicBufferPos] = curMatch; + MOVE_POS_RET; + } + break; + } + + GET_MATCHES_FOOTER_HC(maxLen); } + +static UInt32 * Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +{ + UInt32 mmm; + UInt32 h2, h3, d2, d3, maxLen, pos; + UInt32 *hash; + GET_MATCHES_HEADER(5) + + HASH5_CALC; + + hash = p->hash; + pos = p->pos; + + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + // d4 = pos - (hash + kFix4HashSize)[h4]; + + curMatch = (hash + kFix5HashSize)[hv]; + + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + // (hash + kFix4HashSize)[h4] = pos; + (hash + kFix5HashSize)[hv] = pos; + + SET_mmm + + maxLen = 4; + + for (;;) + { + if (d2 < mmm && *(cur - d2) == *cur) + { + distances[0] = 2; + distances[1] = d2 - 1; + distances += 2; + if (*(cur - d2 + 2) == cur[2]) + { + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + distances[1] = d3 - 1; + distances += 2; + d2 = d3; + } + else + break; + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + distances[1] = d3 - 1; + distances += 2; + d2 = d3; + } + else + break; + + distances[-2] = 3; + if (*(cur - d2 + 3) != cur[3]) + break; + UPDATE_maxLen + distances[-2] = maxLen; + if (maxLen == lenLimit) + { + p->son[p->cyclicBufferPos] = curMatch; + MOVE_POS_RET; + } + break; + } + + GET_MATCHES_FOOTER_HC(maxLen); +} + + +UInt32* Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +{ + GET_MATCHES_HEADER(3) + HASH_ZIP_CALC; + curMatch = p->hash[hv]; + p->hash[hv] = p->pos; + GET_MATCHES_FOOTER_HC(2) +} + + static void Bt2_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { - SKIP_HEADER(2) - HASH2_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - SKIP_FOOTER - } while (--num != 0); + SKIP_HEADER(2) + { + HASH2_CALC; + curMatch = p->hash[hv]; + p->hash[hv] = p->pos; + } + SKIP_FOOTER } void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { - SKIP_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - SKIP_FOOTER - } while (--num != 0); + SKIP_HEADER(3) + { + HASH_ZIP_CALC; + curMatch = p->hash[hv]; + p->hash[hv] = p->pos; + } + SKIP_FOOTER } static void Bt3_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { - UInt32 hash2Value; - SKIP_HEADER(3) - HASH3_CALC; - curMatch = p->hash[kFix3HashSize + hashValue]; - p->hash[hash2Value] = - p->hash[kFix3HashSize + hashValue] = p->pos; - SKIP_FOOTER - } while (--num != 0); + SKIP_HEADER(3) + { + UInt32 h2; + UInt32 *hash; + HASH3_CALC; + hash = p->hash; + curMatch = (hash + kFix3HashSize)[hv]; + hash[h2] = + (hash + kFix3HashSize)[hv] = p->pos; + } + SKIP_FOOTER } static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { - UInt32 hash2Value, hash3Value; - SKIP_HEADER(4) - HASH4_CALC; - curMatch = p->hash[kFix4HashSize + hashValue]; - p->hash[hash2Value] = - p->hash[kFix3HashSize + hash3Value] = p->pos; - p->hash[kFix4HashSize + hashValue] = p->pos; - SKIP_FOOTER - } while (--num != 0); + SKIP_HEADER(4) + { + UInt32 h2, h3; + UInt32 *hash; + HASH4_CALC; + hash = p->hash; + curMatch = (hash + kFix4HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[hv] = p->pos; + } + SKIP_FOOTER } +static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +{ + SKIP_HEADER(5) + { + UInt32 h2, h3; + UInt32 *hash; + HASH5_CALC; + hash = p->hash; + curMatch = (hash + kFix5HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + // (hash + kFix4HashSize)[h4] = + (hash + kFix5HashSize)[hv] = p->pos; + } + SKIP_FOOTER +} + + +#define HC_SKIP_HEADER(minLen) \ + do { if (p->lenLimit < minLen) { MatchFinder_MovePos(p); num--; continue; } { \ + Byte *cur; \ + UInt32 *hash; \ + UInt32 *son; \ + UInt32 pos = p->pos; \ + UInt32 num2 = num; \ + /* (p->pos == p->posLimit) is not allowed here !!! */ \ + { const UInt32 rem = p->posLimit - pos; if (num2 > rem) num2 = rem; } \ + num -= num2; \ + { const UInt32 cycPos = p->cyclicBufferPos; \ + son = p->son + cycPos; \ + p->cyclicBufferPos = cycPos + num2; } \ + cur = p->buffer; \ + hash = p->hash; \ + do { \ + UInt32 curMatch; \ + UInt32 hv; + + +#define HC_SKIP_FOOTER \ + cur++; pos++; *son++ = curMatch; \ + } while (--num2); \ + p->buffer = cur; \ + p->pos = pos; \ + if (pos == p->posLimit) MatchFinder_CheckLimits(p); \ + }} while(num); \ + + static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { - UInt32 hash2Value, hash3Value; - SKIP_HEADER(4) - HASH4_CALC; - curMatch = p->hash[kFix4HashSize + hashValue]; - p->hash[hash2Value] = - p->hash[kFix3HashSize + hash3Value] = - p->hash[kFix4HashSize + hashValue] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } while (--num != 0); + HC_SKIP_HEADER(4) + + UInt32 h2, h3; + HASH4_CALC; + curMatch = (hash + kFix4HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[hv] = pos; + + HC_SKIP_FOOTER } + +static void Hc5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +{ + HC_SKIP_HEADER(5) + + UInt32 h2, h3; + HASH5_CALC + curMatch = (hash + kFix5HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + // (hash + kFix4HashSize)[h4] = + (hash + kFix5HashSize)[hv] = pos; + + HC_SKIP_FOOTER +} + + void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { - SKIP_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } while (--num != 0); + HC_SKIP_HEADER(3) + + HASH_ZIP_CALC; + curMatch = hash[hv]; + hash[hv] = pos; + + HC_SKIP_FOOTER } -void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable) + +void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder2 *vTable) { - vTable->Init = (Mf_Init_Func)MatchFinder_Init; - vTable->GetIndexByte = (Mf_GetIndexByte_Func)MatchFinder_GetIndexByte; - vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinder_GetNumAvailableBytes; - vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinder_GetPointerToCurrentPos; - if (!p->btMode) + vTable->Init = (Mf_Init_Func)MatchFinder_Init; + vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinder_GetNumAvailableBytes; + vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinder_GetPointerToCurrentPos; + if (!p->btMode) + { + if (p->numHashBytes <= 4) { - vTable->GetMatches = (Mf_GetMatches_Func)Hc4_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Hc4_MatchFinder_Skip; - } - else if (p->numHashBytes == 2) - { - vTable->GetMatches = (Mf_GetMatches_Func)Bt2_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt2_MatchFinder_Skip; - } - else if (p->numHashBytes == 3) - { - vTable->GetMatches = (Mf_GetMatches_Func)Bt3_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt3_MatchFinder_Skip; + vTable->GetMatches = (Mf_GetMatches_Func)Hc4_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Hc4_MatchFinder_Skip; } else { - vTable->GetMatches = (Mf_GetMatches_Func)Bt4_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt4_MatchFinder_Skip; + vTable->GetMatches = (Mf_GetMatches_Func)Hc5_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Hc5_MatchFinder_Skip; } -} \ No newline at end of file + } + else if (p->numHashBytes == 2) + { + vTable->GetMatches = (Mf_GetMatches_Func)Bt2_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Bt2_MatchFinder_Skip; + } + else if (p->numHashBytes == 3) + { + vTable->GetMatches = (Mf_GetMatches_Func)Bt3_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Bt3_MatchFinder_Skip; + } + else if (p->numHashBytes == 4) + { + vTable->GetMatches = (Mf_GetMatches_Func)Bt4_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Bt4_MatchFinder_Skip; + } + else + { + vTable->GetMatches = (Mf_GetMatches_Func)Bt5_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Bt5_MatchFinder_Skip; + } +} + + + +void LzFindPrepare() +{ + #ifndef FORCE_SATUR_SUB_128 + #ifdef USE_SATUR_SUB_128 + LZFIND_SATUR_SUB_CODE_FUNC f = NULL; + #ifdef MY_CPU_ARM_OR_ARM64 + { + if (CPU_IsSupported_NEON()) + { + // #pragma message ("=== LzFind NEON") + _PRF(printf("\n=== LzFind NEON\n")); + f = LzFind_SaturSub_128; + } + // f = 0; // for debug + } + #else // MY_CPU_ARM_OR_ARM64 + if (CPU_IsSupported_SSE41()) + { + // #pragma message ("=== LzFind SSE41") + _PRF(printf("\n=== LzFind SSE41\n")); + f = LzFind_SaturSub_128; + + #ifdef USE_AVX2 + if (CPU_IsSupported_AVX2()) + { + // #pragma message ("=== LzFind AVX2") + _PRF(printf("\n=== LzFind AVX2\n")); + f = LzFind_SaturSub_256; + } + #endif + } + #endif // MY_CPU_ARM_OR_ARM64 + g_LzFind_SaturSub = f; + #endif // USE_SATUR_SUB_128 + #endif // FORCE_SATUR_SUB_128 +} diff --git a/common/LZMA/SDK/C/LzFind.h b/common/LZMA/SDK/C/LzFind.h index 010c4b9..eea873f 100644 --- a/common/LZMA/SDK/C/LzFind.h +++ b/common/LZMA/SDK/C/LzFind.h @@ -1,14 +1,12 @@ /* LzFind.h -- Match finder for LZ algorithms -2009-04-22 : Igor Pavlov : Public domain */ +2021-07-13 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_H #define __LZ_FIND_H -#include "Types.h" +#include "7zTypes.h" -#ifdef __cplusplus -extern "C" { -#endif +EXTERN_C_BEGIN typedef UInt32 CLzRef; @@ -17,12 +15,17 @@ typedef struct _CMatchFinder Byte *buffer; UInt32 pos; UInt32 posLimit; - UInt32 streamPos; + UInt32 streamPos; /* wrap over Zero is allowed (streamPos < pos). Use (UInt32)(streamPos - pos) */ UInt32 lenLimit; UInt32 cyclicBufferPos; UInt32 cyclicBufferSize; /* it must be = (historySize + 1) */ + Byte streamEndWasReached; + Byte btMode; + Byte bigHash; + Byte directInput; + UInt32 matchMaxLen; CLzRef *hash; CLzRef *son; @@ -31,32 +34,36 @@ typedef struct _CMatchFinder Byte *bufferBase; ISeqInStream *stream; - int streamEndWasReached; - + UInt32 blockSize; UInt32 keepSizeBefore; UInt32 keepSizeAfter; UInt32 numHashBytes; - int directInput; size_t directInputRem; - int btMode; - int bigHash; UInt32 historySize; UInt32 fixedHashSize; UInt32 hashSizeSum; - UInt32 numSons; SRes result; UInt32 crc[256]; + size_t numRefs; + + UInt64 expectedDataSize; } CMatchFinder; -#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer) -#define Inline_MatchFinder_GetIndexByte(p, index) ((p)->buffer[(Int32)(index)]) +#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((const Byte *)(p)->buffer) -#define Inline_MatchFinder_GetNumAvailableBytes(p) ((p)->streamPos - (p)->pos) +#define Inline_MatchFinder_GetNumAvailableBytes(p) ((UInt32)((p)->streamPos - (p)->pos)) +/* +#define Inline_MatchFinder_IsFinishedOK(p) \ + ((p)->streamEndWasReached \ + && (p)->streamPos == (p)->pos \ + && (!(p)->directInput || (p)->directInputRem == 0)) +*/ + int MatchFinder_NeedMove(CMatchFinder *p); -Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p); +/* Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p); */ void MatchFinder_MoveBlock(CMatchFinder *p); void MatchFinder_ReadIfRequired(CMatchFinder *p); @@ -68,13 +75,24 @@ void MatchFinder_Construct(CMatchFinder *p); */ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAlloc *alloc); -void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc); -void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, UInt32 numItems); -void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); + ISzAllocPtr alloc); +void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc); +void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems); +// void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); + +/* +#define Inline_MatchFinder_InitPos(p, val) \ + (p)->pos = (val); \ + (p)->streamPos = (val); +*/ + +#define Inline_MatchFinder_ReduceOffsets(p, subValue) \ + (p)->pos -= (subValue); \ + (p)->streamPos -= (subValue); + UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *buffer, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, UInt32 *distances, UInt32 maxLen); /* @@ -84,32 +102,35 @@ Conditions: */ typedef void (*Mf_Init_Func)(void *object); -typedef Byte (*Mf_GetIndexByte_Func)(void *object, Int32 index); typedef UInt32 (*Mf_GetNumAvailableBytes_Func)(void *object); typedef const Byte * (*Mf_GetPointerToCurrentPos_Func)(void *object); -typedef UInt32 (*Mf_GetMatches_Func)(void *object, UInt32 *distances); +typedef UInt32 * (*Mf_GetMatches_Func)(void *object, UInt32 *distances); typedef void (*Mf_Skip_Func)(void *object, UInt32); typedef struct _IMatchFinder { Mf_Init_Func Init; - Mf_GetIndexByte_Func GetIndexByte; Mf_GetNumAvailableBytes_Func GetNumAvailableBytes; Mf_GetPointerToCurrentPos_Func GetPointerToCurrentPos; Mf_GetMatches_Func GetMatches; Mf_Skip_Func Skip; -} IMatchFinder; +} IMatchFinder2; -void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable); +void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder2 *vTable); +void MatchFinder_Init_LowHash(CMatchFinder *p); +void MatchFinder_Init_HighHash(CMatchFinder *p); +void MatchFinder_Init_4(CMatchFinder *p); void MatchFinder_Init(CMatchFinder *p); -UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); -UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); + +UInt32* Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); +UInt32* Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); + void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); -#ifdef __cplusplus -} -#endif +void LzFindPrepare(void); + +EXTERN_C_END #endif diff --git a/common/LZMA/SDK/C/LzHash.h b/common/LZMA/SDK/C/LzHash.h index f3e8996..77b898c 100644 --- a/common/LZMA/SDK/C/LzHash.h +++ b/common/LZMA/SDK/C/LzHash.h @@ -1,54 +1,34 @@ /* LzHash.h -- HASH functions for LZ algorithms -2009-02-07 : Igor Pavlov : Public domain */ +2019-10-30 : Igor Pavlov : Public domain */ #ifndef __LZ_HASH_H #define __LZ_HASH_H +/* + (kHash2Size >= (1 << 8)) : Required + (kHash3Size >= (1 << 16)) : Required +*/ + #define kHash2Size (1 << 10) #define kHash3Size (1 << 16) -#define kHash4Size (1 << 20) +// #define kHash4Size (1 << 20) #define kFix3HashSize (kHash2Size) #define kFix4HashSize (kHash2Size + kHash3Size) -#define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) +// #define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) -#define HASH2_CALC hashValue = cur[0] | ((UInt32)cur[1] << 8); +/* + We use up to 3 crc values for hash: + crc0 + crc1 << Shift_1 + crc2 << Shift_2 + (Shift_1 = 5) and (Shift_2 = 10) is good tradeoff. + Small values for Shift are not good for collision rate. + Big value for Shift_2 increases the minimum size + of hash table, that will be slow for small files. +*/ -#define HASH3_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - hash2Value = temp & (kHash2Size - 1); \ - hashValue = (temp ^ ((UInt32)cur[2] << 8)) & p->hashMask; } - -#define HASH4_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - hash2Value = temp & (kHash2Size - 1); \ - hash3Value = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); \ - hashValue = (temp ^ ((UInt32)cur[2] << 8) ^ (p->crc[cur[3]] << 5)) & p->hashMask; } - -#define HASH5_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - hash2Value = temp & (kHash2Size - 1); \ - hash3Value = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); \ - hash4Value = (temp ^ ((UInt32)cur[2] << 8) ^ (p->crc[cur[3]] << 5)); \ - hashValue = (hash4Value ^ (p->crc[cur[4]] << 3)) & p->hashMask; \ - hash4Value &= (kHash4Size - 1); } - -/* #define HASH_ZIP_CALC hashValue = ((cur[0] | ((UInt32)cur[1] << 8)) ^ p->crc[cur[2]]) & 0xFFFF; */ -#define HASH_ZIP_CALC hashValue = ((cur[2] | ((UInt32)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF; - - -#define MT_HASH2_CALC \ - hash2Value = (p->crc[cur[0]] ^ cur[1]) & (kHash2Size - 1); - -#define MT_HASH3_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - hash2Value = temp & (kHash2Size - 1); \ - hash3Value = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } - -#define MT_HASH4_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - hash2Value = temp & (kHash2Size - 1); \ - hash3Value = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); \ - hash4Value = (temp ^ ((UInt32)cur[2] << 8) ^ (p->crc[cur[3]] << 5)) & (kHash4Size - 1); } +#define kLzHash_CrcShift_1 5 +#define kLzHash_CrcShift_2 10 #endif diff --git a/common/LZMA/SDK/C/LzmaDec.c b/common/LZMA/SDK/C/LzmaDec.c index 9be3eec..d6742e5 100644 --- a/common/LZMA/SDK/C/LzmaDec.c +++ b/common/LZMA/SDK/C/LzmaDec.c @@ -1,32 +1,44 @@ /* LzmaDec.c -- LZMA Decoder -2009-09-20 : Igor Pavlov : Public domain*/ +2021-04-01 : Igor Pavlov : Public domain */ -#include "LzmaDec.h" +#include "Precomp.h" #include +/* #include "CpuArch.h" */ +#include "LzmaDec.h" + #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits) #define kNumBitModelTotalBits 11 #define kBitModelTotal (1 << kNumBitModelTotalBits) -#define kNumMoveBits 5 #define RC_INIT_SIZE 5 +#ifndef _LZMA_DEC_OPT + +#define kNumMoveBits 5 #define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); #define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); #define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ - { UPDATE_0(p); i = (i + i); A0; } else \ - { UPDATE_1(p); i = (i + i) + 1; A1; } -#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;) + { UPDATE_0(p); i = (i + i); A0; } else \ + { UPDATE_1(p); i = (i + i) + 1; A1; } + +#define TREE_GET_BIT(probs, i) { GET_BIT2(probs + i, i, ;, ;); } + +#define REV_BIT(p, i, A0, A1) IF_BIT_0(p + i) \ + { UPDATE_0(p + i); A0; } else \ + { UPDATE_1(p + i); A1; } +#define REV_BIT_VAR( p, i, m) REV_BIT(p, i, i += m; m += m, m += m; i += m; ) +#define REV_BIT_CONST(p, i, m) REV_BIT(p, i, i += m; , i += m * 2; ) +#define REV_BIT_LAST( p, i, m) REV_BIT(p, i, i -= m , ; ) -#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); } #define TREE_DECODE(probs, limit, i) \ - { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } + { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } /* #define _LZMA_SIZE_OPT */ @@ -34,7 +46,7 @@ #define TREE_6_DECODE(probs, i) TREE_DECODE(probs, (1 << 6), i) #else #define TREE_6_DECODE(probs, i) \ - { i = 1; \ + { i = 1; \ TREE_GET_BIT(probs, i); \ TREE_GET_BIT(probs, i); \ TREE_GET_BIT(probs, i); \ @@ -44,36 +56,52 @@ i -= 0x40; } #endif -#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } +#define NORMAL_LITER_DEC TREE_GET_BIT(prob, symbol) +#define MATCHED_LITER_DEC \ + matchByte += matchByte; \ + bit = offs; \ + offs &= matchByte; \ + probLit = prob + (offs + bit + symbol); \ + GET_BIT2(probLit, symbol, offs ^= bit; , ;) -#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#endif // _LZMA_DEC_OPT + + +#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_INPUT_EOF; range <<= 8; code = (code << 8) | (*buf++); } + +#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0_CHECK range = bound; #define UPDATE_1_CHECK range -= bound; code -= bound; #define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \ - { UPDATE_0_CHECK; i = (i + i); A0; } else \ - { UPDATE_1_CHECK; i = (i + i) + 1; A1; } + { UPDATE_0_CHECK; i = (i + i); A0; } else \ + { UPDATE_1_CHECK; i = (i + i) + 1; A1; } #define GET_BIT_CHECK(p, i) GET_BIT2_CHECK(p, i, ; , ;) #define TREE_DECODE_CHECK(probs, limit, i) \ - { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } + { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } + + +#define REV_BIT_CHECK(p, i, m) IF_BIT_0_CHECK(p + i) \ + { UPDATE_0_CHECK; i += m; m += m; } else \ + { UPDATE_1_CHECK; m += m; i += m; } + #define kNumPosBitsMax 4 #define kNumPosStatesMax (1 << kNumPosBitsMax) #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) -#define LenChoice 0 -#define LenChoice2 (LenChoice + 1) -#define LenLow (LenChoice2 + 1) -#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) -#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define LenLow 0 +#define LenHigh (LenLow + 2 * (kNumPosStatesMax << kLenNumLowBits)) #define kNumLenProbs (LenHigh + kLenNumHighSymbols) +#define LenChoice LenLow +#define LenChoice2 (LenLow + (1 << kLenNumLowBits)) + #define kNumStates 12 +#define kNumStates2 16 #define kNumLitStates 7 #define kStartPosModelIndex 4 @@ -87,901 +115,1249 @@ #define kAlignTableSize (1 << kNumAlignBits) #define kMatchMinLen 2 -#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) -#define IsMatch 0 -#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +#define kMatchSpecLen_Error_Data (1 << 9) +#define kMatchSpecLen_Error_Fail (kMatchSpecLen_Error_Data - 1) + +/* External ASM code needs same CLzmaProb array layout. So don't change it. */ + +/* (probs_1664) is faster and better for code size at some platforms */ +/* +#ifdef MY_CPU_X86_OR_AMD64 +*/ +#define kStartOffset 1664 +#define GET_PROBS p->probs_1664 +/* +#define GET_PROBS p->probs + kStartOffset +#else +#define kStartOffset 0 +#define GET_PROBS p->probs +#endif +*/ + +#define SpecPos (-kStartOffset) +#define IsRep0Long (SpecPos + kNumFullDistances) +#define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) +#define LenCoder (RepLenCoder + kNumLenProbs) +#define IsMatch (LenCoder + kNumLenProbs) +#define Align (IsMatch + (kNumStates2 << kNumPosBitsMax)) +#define IsRep (Align + kAlignTableSize) #define IsRepG0 (IsRep + kNumStates) #define IsRepG1 (IsRepG0 + kNumStates) #define IsRepG2 (IsRepG1 + kNumStates) -#define IsRep0Long (IsRepG2 + kNumStates) -#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) -#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) -#define LenCoder (Align + kAlignTableSize) -#define RepLenCoder (LenCoder + kNumLenProbs) -#define Literal (RepLenCoder + kNumLenProbs) +#define PosSlot (IsRepG2 + kNumStates) +#define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define NUM_BASE_PROBS (Literal + kStartOffset) -#define LZMA_BASE_SIZE 1846 -#define LZMA_LIT_SIZE 768 - -#define LzmaProps_GetNumProbs(p) ((UInt32)LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) - -#if Literal != LZMA_BASE_SIZE -StopCompilingDueBUG +#if Align != 0 && kStartOffset != 0 + #error Stop_Compiling_Bad_LZMA_kAlign #endif +#if NUM_BASE_PROBS != 1984 + #error Stop_Compiling_Bad_LZMA_PROBS +#endif + + +#define LZMA_LIT_SIZE 0x300 + +#define LzmaProps_GetNumProbs(p) (NUM_BASE_PROBS + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) + + +#define CALC_POS_STATE(processedPos, pbMask) (((processedPos) & (pbMask)) << 4) +#define COMBINED_PS_STATE (posState + state) +#define GET_LEN_STATE (posState) + #define LZMA_DIC_MIN (1 << 12) -/* First LZMA-symbol is always decoded. -And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is with last normalization -Out: -Result: -SZ_OK - OK -SZ_ERROR_DATA - Error -p->remainLen: -< kMatchSpecLenStart : normal remain -= kMatchSpecLenStart : finished -= kMatchSpecLenStart + 1 : Flush marker -= kMatchSpecLenStart + 2 : State Init Marker +/* +p->remainLen : shows status of LZMA decoder: + < kMatchSpecLenStart : the number of bytes to be copied with (p->rep0) offset + = kMatchSpecLenStart : the LZMA stream was finished with end mark + = kMatchSpecLenStart + 1 : need init range coder + = kMatchSpecLenStart + 2 : need init range coder and state + = kMatchSpecLen_Error_Fail : Internal Code Failure + = kMatchSpecLen_Error_Data + [0 ... 273] : LZMA Data Error */ -static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +/* ---------- LZMA_DECODE_REAL ---------- */ +/* +LzmaDec_DecodeReal_3() can be implemented in external ASM file. +3 - is the code compatibility version of that function for check at link time. +*/ + +#define LZMA_DECODE_REAL LzmaDec_DecodeReal_3 + +/* +LZMA_DECODE_REAL() +In: + RangeCoder is normalized + if (p->dicPos == limit) + { + LzmaDec_TryDummy() was called before to exclude LITERAL and MATCH-REP cases. + So first symbol can be only MATCH-NON-REP. And if that MATCH-NON-REP symbol + is not END_OF_PAYALOAD_MARKER, then the function doesn't write any byte to dictionary, + the function returns SZ_OK, and the caller can use (p->remainLen) and (p->reps[0]) later. + } + +Processing: + The first LZMA symbol will be decoded in any case. + All main checks for limits are at the end of main loop, + It decodes additional LZMA-symbols while (p->buf < bufLimit && dicPos < limit), + RangeCoder is still without last normalization when (p->buf < bufLimit) is being checked. + But if (p->buf < bufLimit), the caller provided at least (LZMA_REQUIRED_INPUT_MAX + 1) bytes for + next iteration before limit (bufLimit + LZMA_REQUIRED_INPUT_MAX), + that is enough for worst case LZMA symbol with one additional RangeCoder normalization for one bit. + So that function never reads bufLimit [LZMA_REQUIRED_INPUT_MAX] byte. + +Out: + RangeCoder is normalized + Result: + SZ_OK - OK + p->remainLen: + < kMatchSpecLenStart : the number of bytes to be copied with (p->reps[0]) offset + = kMatchSpecLenStart : the LZMA stream was finished with end mark + + SZ_ERROR_DATA - error, when the MATCH-Symbol refers out of dictionary + p->remainLen : undefined + p->reps[*] : undefined +*/ + + +#ifdef _LZMA_DEC_OPT + +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit); + +#else + +static +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { - CLzmaProb *probs = p->probs; + CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; + UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; + unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; + unsigned lc = p->prop.lc; + unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); - unsigned state = p->state; - UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; - unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; - unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1; - unsigned lc = p->prop.lc; + Byte *dic = p->dic; + SizeT dicBufSize = p->dicBufSize; + SizeT dicPos = p->dicPos; + + UInt32 processedPos = p->processedPos; + UInt32 checkDicSize = p->checkDicSize; + unsigned len = 0; - Byte *dic = p->dic; - SizeT dicBufSize = p->dicBufSize; - SizeT dicPos = p->dicPos; + const Byte *buf = p->buf; + UInt32 range = p->range; + UInt32 code = p->code; - UInt32 processedPos = p->processedPos; - UInt32 checkDicSize = p->checkDicSize; - unsigned len = 0; + do + { + CLzmaProb *prob; + UInt32 bound; + unsigned ttt; + unsigned posState = CALC_POS_STATE(processedPos, pbMask); - const Byte *buf = p->buf; - UInt32 range = p->range; - UInt32 code = p->code; - - do - { - CLzmaProb *prob; - UInt32 bound; - unsigned ttt; - unsigned posState = processedPos & pbMask; - - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; - IF_BIT_0(prob) - { - unsigned symbol; - UPDATE_0(prob); - prob = probs + Literal; - if (checkDicSize != 0 || processedPos != 0) - prob += (LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) + - (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc)))); - - if (state < kNumLitStates) - { - state -= (state < 4) ? state : 3; - symbol = 1; - do { GET_BIT(prob + symbol, symbol) } while (symbol < 0x100); - } - else - { - unsigned matchByte = p->dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)]; - unsigned offs = 0x100; - state -= (state < 10) ? 3 : 6; - symbol = 1; - do - { - unsigned bit; - CLzmaProb *probLit; - matchByte <<= 1; - bit = (matchByte & offs); - probLit = prob + offs + bit + symbol; - GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit) - } while (symbol < 0x100); - } - dic[dicPos++] = (Byte)symbol; - processedPos++; - continue; - } -else -{ - UPDATE_1(prob); - prob = probs + IsRep + state; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0(prob) { + unsigned symbol; + UPDATE_0(prob); + prob = probs + Literal; + if (processedPos != 0 || checkDicSize != 0) + prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); + processedPos++; + + if (state < kNumLitStates) + { + state -= (state < 4) ? state : 3; + symbol = 1; + #ifdef _LZMA_SIZE_OPT + do { NORMAL_LITER_DEC } while (symbol < 0x100); + #else + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + #endif + } + else + { + unsigned matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + unsigned offs = 0x100; + state -= (state < 10) ? 3 : 6; + symbol = 1; + #ifdef _LZMA_SIZE_OPT + do + { + unsigned bit; + CLzmaProb *probLit; + MATCHED_LITER_DEC + } + while (symbol < 0x100); + #else + { + unsigned bit; + CLzmaProb *probLit; + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + } + #endif + } + + dic[dicPos++] = (Byte)symbol; + continue; + } + + { + UPDATE_1(prob); + prob = probs + IsRep + state; + IF_BIT_0(prob) + { UPDATE_0(prob); state += kNumStates; prob = probs + LenCoder; - } -else -{ - UPDATE_1(prob); - if (checkDicSize == 0 && processedPos == 0) - return SZ_ERROR_DATA; - prob = probs + IsRepG0 + state; - IF_BIT_0(prob) - { - UPDATE_0(prob); - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + } + else + { + UPDATE_1(prob); + prob = probs + IsRepG0 + state; IF_BIT_0(prob) { + UPDATE_0(prob); + prob = probs + IsRep0Long + COMBINED_PS_STATE; + IF_BIT_0(prob) + { UPDATE_0(prob); - dic[dicPos] = dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)]; + + // that case was checked before with kBadRepCode + // if (checkDicSize == 0 && processedPos == 0) { len = kMatchSpecLen_Error_Data + 1; break; } + // The caller doesn't allow (dicPos == limit) case here + // so we don't need the following check: + // if (dicPos == limit) { state = state < kNumLitStates ? 9 : 11; len = 1; break; } + + dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; dicPos++; processedPos++; state = state < kNumLitStates ? 9 : 11; continue; + } + UPDATE_1(prob); } - UPDATE_1(prob); - } - else - { - UInt32 distance; - UPDATE_1(prob); - prob = probs + IsRepG1 + state; - IF_BIT_0(prob) + else { + UInt32 distance; + UPDATE_1(prob); + prob = probs + IsRepG1 + state; + IF_BIT_0(prob) + { UPDATE_0(prob); distance = rep1; - } + } else { - UPDATE_1(prob); - prob = probs + IsRepG2 + state; - IF_BIT_0(prob) - { - UPDATE_0(prob); - distance = rep2; - } + UPDATE_1(prob); + prob = probs + IsRepG2 + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + distance = rep2; + } else { - UPDATE_1(prob); - distance = rep3; - rep3 = rep2; + UPDATE_1(prob); + distance = rep3; + rep3 = rep2; } rep2 = rep1; } rep1 = rep0; rep0 = distance; - } - state = state < kNumLitStates ? 8 : 11; - prob = probs + RepLenCoder; -} + } + state = state < kNumLitStates ? 8 : 11; + prob = probs + RepLenCoder; + } + + #ifdef _LZMA_SIZE_OPT { - unsigned limit, offset; - CLzmaProb *probLen = prob + LenChoice; + unsigned lim, offset; + CLzmaProb *probLen = prob + LenChoice; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenLow + GET_LEN_STATE; + offset = 0; + lim = (1 << kLenNumLowBits); + } + else + { + UPDATE_1(probLen); + probLen = prob + LenChoice2; IF_BIT_0(probLen) { - UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); - offset = 0; - limit = (1 << kLenNumLowBits); + UPDATE_0(probLen); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); + offset = kLenNumLowSymbols; + lim = (1 << kLenNumLowBits); } - else - { - UPDATE_1(probLen); - probLen = prob + LenChoice2; - IF_BIT_0(probLen) - { - UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); - offset = kLenNumLowSymbols; - limit = (1 << kLenNumMidBits); - } else { - UPDATE_1(probLen); - probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; - limit = (1 << kLenNumHighBits); + UPDATE_1(probLen); + probLen = prob + LenHigh; + offset = kLenNumLowSymbols * 2; + lim = (1 << kLenNumHighBits); } } - TREE_DECODE(probLen, limit, len); + TREE_DECODE(probLen, lim, len); len += offset; } - -if (state >= kNumStates) -{ - UInt32 distance; - prob = probs + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); - TREE_6_DECODE(prob, distance); - if (distance >= kStartPosModelIndex) - { - unsigned posSlot = (unsigned)distance; - int numDirectBits = (int)(((distance >> 1) - 1)); - distance = (2 | (distance & 1)); - if (posSlot < kEndPosModelIndex) + #else + { + CLzmaProb *probLen = prob + LenChoice; + IF_BIT_0(probLen) { - distance <<= numDirectBits; - prob = probs + SpecPos + distance - posSlot - 1; - { - UInt32 mask = 1; - unsigned i = 1; - do - { - GET_BIT2(prob + i, i, ;, distance |= mask); - mask <<= 1; - } while (--numDirectBits != 0); - } + UPDATE_0(probLen); + probLen = prob + LenLow + GET_LEN_STATE; + len = 1; + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + len -= 8; } else { + UPDATE_1(probLen); + probLen = prob + LenChoice2; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); + len = 1; + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + } + else + { + UPDATE_1(probLen); + probLen = prob + LenHigh; + TREE_DECODE(probLen, (1 << kLenNumHighBits), len); + len += kLenNumLowSymbols * 2; + } + } + } + #endif + + if (state >= kNumStates) + { + UInt32 distance; + prob = probs + PosSlot + + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); + TREE_6_DECODE(prob, distance); + if (distance >= kStartPosModelIndex) + { + unsigned posSlot = (unsigned)distance; + unsigned numDirectBits = (unsigned)(((distance >> 1) - 1)); + distance = (2 | (distance & 1)); + if (posSlot < kEndPosModelIndex) + { + distance <<= numDirectBits; + prob = probs + SpecPos; + { + UInt32 m = 1; + distance++; + do + { + REV_BIT_VAR(prob, distance, m); + } + while (--numDirectBits); + distance -= m; + } + } + else + { numDirectBits -= kNumAlignBits; do { - NORMALIZE - range >>= 1; - - { - UInt32 t; - code -= range; - t = (0 - ((UInt32)code >> 31)); /* (UInt32)((Int32)code >> 31) */ - distance = (distance << 1) + (t + 1); - code += range & t; - } - /* - distance <<= 1; - if (code >= range) - { + NORMALIZE + range >>= 1; + + { + UInt32 t; + code -= range; + t = (0 - ((UInt32)code >> 31)); /* (UInt32)((Int32)code >> 31) */ + distance = (distance << 1) + (t + 1); + code += range & t; + } + /* + distance <<= 1; + if (code >= range) + { code -= range; distance |= 1; - } - */ - } while (--numDirectBits != 0); + } + */ + } + while (--numDirectBits); prob = probs + Align; distance <<= kNumAlignBits; { - unsigned i = 1; - GET_BIT2(prob + i, i, ;, distance |= 1); - GET_BIT2(prob + i, i, ;, distance |= 2); - GET_BIT2(prob + i, i, ;, distance |= 4); - GET_BIT2(prob + i, i, ;, distance |= 8); + unsigned i = 1; + REV_BIT_CONST(prob, i, 1); + REV_BIT_CONST(prob, i, 2); + REV_BIT_CONST(prob, i, 4); + REV_BIT_LAST (prob, i, 8); + distance |= i; } if (distance == (UInt32)0xFFFFFFFF) { - len += kMatchSpecLenStart; - state -= kNumStates; - break; + len = kMatchSpecLenStart; + state -= kNumStates; + break; } + } } - } - rep3 = rep2; - rep2 = rep1; - rep1 = rep0; - rep0 = distance + 1; - if (checkDicSize == 0) - { - if (distance >= processedPos) - return SZ_ERROR_DATA; - } - else if (distance >= checkDicSize) - return SZ_ERROR_DATA; - state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; -} - -len += kMatchMinLen; - -if (limit == dicPos) -return SZ_ERROR_DATA; -{ - SizeT rem = limit - dicPos; - unsigned curLen = ((rem < len) ? (unsigned)rem : len); - SizeT pos = (dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0); - - processedPos += curLen; - - len -= curLen; - if (pos + curLen <= dicBufSize) - { - Byte *dest = dic + dicPos; - ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; - const Byte *lim = dest + curLen; - dicPos += curLen; - do - *(dest) = (Byte)*(dest + src); - while (++dest != lim); - } - else - { - do + + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance + 1; + state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) { + len += kMatchSpecLen_Error_Data + kMatchMinLen; + // len = kMatchSpecLen_Error_Data; + // len += kMatchMinLen; + break; + } + } + + len += kMatchMinLen; + + { + SizeT rem; + unsigned curLen; + SizeT pos; + + if ((rem = limit - dicPos) == 0) + { + /* + We stop decoding and return SZ_OK, and we can resume decoding later. + Any error conditions can be tested later in caller code. + For more strict mode we can stop decoding with error + // len += kMatchSpecLen_Error_Data; + */ + break; + } + + curLen = ((rem < len) ? (unsigned)rem : len); + pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); + + processedPos += (UInt32)curLen; + + len -= curLen; + if (curLen <= dicBufSize - pos) + { + Byte *dest = dic + dicPos; + ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; + const Byte *lim = dest + curLen; + dicPos += (SizeT)curLen; + do + *(dest) = (Byte)*(dest + src); + while (++dest != lim); + } + else + { + do + { dic[dicPos++] = dic[pos]; if (++pos == dicBufSize) - pos = 0; - } while (--curLen != 0); + pos = 0; + } + while (--curLen != 0); + } + } } -} -} - } while (dicPos < limit && buf < bufLimit); - NORMALIZE; - p->buf = buf; - p->range = range; - p->code = code; - p->remainLen = len; - p->dicPos = dicPos; - p->processedPos = processedPos; - p->reps[0] = rep0; - p->reps[1] = rep1; - p->reps[2] = rep2; - p->reps[3] = rep3; - p->state = state; + } + while (dicPos < limit && buf < bufLimit); - return SZ_OK; + NORMALIZE; + + p->buf = buf; + p->range = range; + p->code = code; + p->remainLen = (UInt32)len; // & (kMatchSpecLen_Error_Data - 1); // we can write real length for error matches too. + p->dicPos = dicPos; + p->processedPos = processedPos; + p->reps[0] = rep0; + p->reps[1] = rep1; + p->reps[2] = rep2; + p->reps[3] = rep3; + p->state = (UInt32)state; + if (len >= kMatchSpecLen_Error_Data) + return SZ_ERROR_DATA; + return SZ_OK; } +#endif + + static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) { - if (p->remainLen != 0 && p->remainLen < kMatchSpecLenStart) + unsigned len = (unsigned)p->remainLen; + if (len == 0 /* || len >= kMatchSpecLenStart */) + return; + { + SizeT dicPos = p->dicPos; + Byte *dic; + SizeT dicBufSize; + SizeT rep0; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ { - Byte *dic = p->dic; - SizeT dicPos = p->dicPos; - SizeT dicBufSize = p->dicBufSize; - unsigned len = p->remainLen; - UInt32 rep0 = p->reps[0]; - if (limit - dicPos < len) - len = (unsigned)(limit - dicPos); - - if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) - p->checkDicSize = p->prop.dicSize; - - p->processedPos += len; - p->remainLen -= len; - while (len-- != 0) - { - dic[dicPos] = dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)]; - dicPos++; - } - p->dicPos = dicPos; + SizeT rem = limit - dicPos; + if (rem < len) + { + len = (unsigned)(rem); + if (len == 0) + return; + } } + + if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) + p->checkDicSize = p->prop.dicSize; + + p->processedPos += (UInt32)len; + p->remainLen -= (UInt32)len; + dic = p->dic; + rep0 = p->reps[0]; + dicBufSize = p->dicBufSize; + do + { + dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + dicPos++; + } + while (--len); + p->dicPos = dicPos; + } } + +/* +At staring of new stream we have one of the following symbols: + - Literal - is allowed + - Non-Rep-Match - is allowed only if it's end marker symbol + - Rep-Match - is not allowed +We use early check of (RangeCoder:Code) over kBadRepCode to simplify main decoding code +*/ + +#define kRange0 0xFFFFFFFF +#define kBound0 ((kRange0 >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1)) +#define kBadRepCode (kBound0 + (((kRange0 - kBound0) >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1))) +#if kBadRepCode != (0xC0000000 - 0x400) + #error Stop_Compiling_Bad_LZMA_Check +#endif + + +/* +LzmaDec_DecodeReal2(): + It calls LZMA_DECODE_REAL() and it adjusts limit according (p->checkDicSize). + +We correct (p->checkDicSize) after LZMA_DECODE_REAL() and in LzmaDec_WriteRem(), +and we support the following state of (p->checkDicSize): + if (total_processed < p->prop.dicSize) then + { + (total_processed == p->processedPos) + (p->checkDicSize == 0) + } + else + (p->checkDicSize == p->prop.dicSize) +*/ + static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { - do - { - SizeT limit2 = limit; - if (p->checkDicSize == 0) - { - UInt32 rem = p->prop.dicSize - p->processedPos; - if (limit - p->dicPos > rem) - limit2 = p->dicPos + rem; - } - RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit)); - if (p->processedPos >= p->prop.dicSize) - p->checkDicSize = p->prop.dicSize; - LzmaDec_WriteRem(p, limit); - } while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); - - if (p->remainLen > kMatchSpecLenStart) - { - p->remainLen = kMatchSpecLenStart; - } - return 0; + if (p->checkDicSize == 0) + { + UInt32 rem = p->prop.dicSize - p->processedPos; + if (limit - p->dicPos > rem) + limit = p->dicPos + rem; + } + { + int res = LZMA_DECODE_REAL(p, limit, bufLimit); + if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize) + p->checkDicSize = p->prop.dicSize; + return res; + } } + + typedef enum { - DUMMY_ERROR, /* unexpected end of input stream */ - DUMMY_LIT, - DUMMY_MATCH, - DUMMY_REP + DUMMY_INPUT_EOF, /* need more input data */ + DUMMY_LIT, + DUMMY_MATCH, + DUMMY_REP } ELzmaDummy; -static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inSize) + +#define IS_DUMMY_END_MARKER_POSSIBLE(dummyRes) ((dummyRes) == DUMMY_MATCH) + +static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, const Byte **bufOut) { - UInt32 range = p->range; - UInt32 code = p->code; - const Byte *bufLimit = buf + inSize; - CLzmaProb *probs = p->probs; - unsigned state = p->state; - ELzmaDummy res; + UInt32 range = p->range; + UInt32 code = p->code; + const Byte *bufLimit = *bufOut; + const CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; + ELzmaDummy res; - { - CLzmaProb *prob; - UInt32 bound; - unsigned ttt; - unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1); + for (;;) + { + const CLzmaProb *prob; + UInt32 bound; + unsigned ttt; + unsigned posState = CALC_POS_STATE(p->processedPos, ((unsigned)1 << p->prop.pb) - 1); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK - - /* if (bufLimit - buf >= 7) return DUMMY_LIT; */ - - prob = probs + Literal; - if (p->checkDicSize != 0 || p->processedPos != 0) - prob += (LZMA_LIT_SIZE * - ((((p->processedPos) & ((1 << (p->prop.lp)) - 1)) << p->prop.lc) + - (p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc)))); - - if (state < kNumLitStates) - { - unsigned symbol = 1; - do { GET_BIT_CHECK(prob + symbol, symbol) } while (symbol < 0x100); - } - else - { - unsigned matchByte = p->dic[p->dicPos - p->reps[0] + - ((p->dicPos < p->reps[0]) ? p->dicBufSize : 0)]; - unsigned offs = 0x100; - unsigned symbol = 1; - do - { - unsigned bit; - CLzmaProb *probLit; - matchByte <<= 1; - bit = (matchByte & offs); - probLit = prob + offs + bit + symbol; - GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit) - } while (symbol < 0x100); - } - res = DUMMY_LIT; - } -else -{ - unsigned len; - UPDATE_1_CHECK; - - prob = probs + IsRep + state; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { + UPDATE_0_CHECK + + prob = probs + Literal; + if (p->checkDicSize != 0 || p->processedPos != 0) + prob += ((UInt32)LZMA_LIT_SIZE * + ((((p->processedPos) & (((unsigned)1 << (p->prop.lp)) - 1)) << p->prop.lc) + + ((unsigned)p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc)))); + + if (state < kNumLitStates) + { + unsigned symbol = 1; + do { GET_BIT_CHECK(prob + symbol, symbol) } while (symbol < 0x100); + } + else + { + unsigned matchByte = p->dic[p->dicPos - p->reps[0] + + (p->dicPos < p->reps[0] ? p->dicBufSize : 0)]; + unsigned offs = 0x100; + unsigned symbol = 1; + do + { + unsigned bit; + const CLzmaProb *probLit; + matchByte += matchByte; + bit = offs; + offs &= matchByte; + probLit = prob + (offs + bit + symbol); + GET_BIT2_CHECK(probLit, symbol, offs ^= bit; , ; ) + } + while (symbol < 0x100); + } + res = DUMMY_LIT; + } + else + { + unsigned len; + UPDATE_1_CHECK; + + prob = probs + IsRep + state; + IF_BIT_0_CHECK(prob) + { UPDATE_0_CHECK; state = 0; prob = probs + LenCoder; res = DUMMY_MATCH; - } -else -{ - UPDATE_1_CHECK; - res = DUMMY_REP; - prob = probs + IsRepG0 + state; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + } + else + { + UPDATE_1_CHECK; + res = DUMMY_REP; + prob = probs + IsRepG0 + state; IF_BIT_0_CHECK(prob) { + UPDATE_0_CHECK; + prob = probs + IsRep0Long + COMBINED_PS_STATE; + IF_BIT_0_CHECK(prob) + { UPDATE_0_CHECK; - NORMALIZE_CHECK; - return DUMMY_REP; - } + break; + } else { - UPDATE_1_CHECK; + UPDATE_1_CHECK; } - } + } else { - UPDATE_1_CHECK; - prob = probs + IsRepG1 + state; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - } + UPDATE_1_CHECK; + prob = probs + IsRepG1 + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + } else { - UPDATE_1_CHECK; - prob = probs + IsRepG2 + state; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - } + UPDATE_1_CHECK; + prob = probs + IsRepG2 + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + } else { - UPDATE_1_CHECK; + UPDATE_1_CHECK; } } } state = kNumStates; prob = probs + RepLenCoder; -} + } { - unsigned limit, offset; - CLzmaProb *probLen = prob + LenChoice; - IF_BIT_0_CHECK(probLen) - { - UPDATE_0_CHECK; - probLen = prob + LenLow + (posState << kLenNumLowBits); - offset = 0; - limit = 1 << kLenNumLowBits; - } + unsigned limit, offset; + const CLzmaProb *probLen = prob + LenChoice; + IF_BIT_0_CHECK(probLen) + { + UPDATE_0_CHECK; + probLen = prob + LenLow + GET_LEN_STATE; + offset = 0; + limit = 1 << kLenNumLowBits; + } else { - UPDATE_1_CHECK; - probLen = prob + LenChoice2; - IF_BIT_0_CHECK(probLen) - { - UPDATE_0_CHECK; - probLen = prob + LenMid + (posState << kLenNumMidBits); - offset = kLenNumLowSymbols; - limit = 1 << kLenNumMidBits; - } + UPDATE_1_CHECK; + probLen = prob + LenChoice2; + IF_BIT_0_CHECK(probLen) + { + UPDATE_0_CHECK; + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); + offset = kLenNumLowSymbols; + limit = 1 << kLenNumLowBits; + } else { - UPDATE_1_CHECK; - probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; - limit = 1 << kLenNumHighBits; + UPDATE_1_CHECK; + probLen = prob + LenHigh; + offset = kLenNumLowSymbols * 2; + limit = 1 << kLenNumHighBits; } } TREE_DECODE_CHECK(probLen, limit, len); len += offset; } -if (state < 4) -{ - unsigned posSlot; - prob = probs + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << - kNumPosSlotBits); - TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); - if (posSlot >= kStartPosModelIndex) - { - int numDirectBits = ((posSlot >> 1) - 1); - - /* if (bufLimit - buf >= 8) return DUMMY_MATCH; */ - - if (posSlot < kEndPosModelIndex) - { - prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1; - } - else + if (state < 4) + { + unsigned posSlot; + prob = probs + PosSlot + + ((len < kNumLenToPosStates - 1 ? len : kNumLenToPosStates - 1) << + kNumPosSlotBits); + TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); + if (posSlot >= kStartPosModelIndex) { + unsigned numDirectBits = ((posSlot >> 1) - 1); + + if (posSlot < kEndPosModelIndex) + { + prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits); + } + else + { numDirectBits -= kNumAlignBits; do { - NORMALIZE_CHECK - range >>= 1; - code -= range & (((code - range) >> 31) - 1); - /* if (code >= range) code -= range; */ - } while (--numDirectBits != 0); + NORMALIZE_CHECK + range >>= 1; + code -= range & (((code - range) >> 31) - 1); + /* if (code >= range) code -= range; */ + } + while (--numDirectBits); prob = probs + Align; numDirectBits = kNumAlignBits; - } - { - unsigned i = 1; - do - { - GET_BIT_CHECK(prob + i, i); - } while (--numDirectBits != 0); } + { + unsigned i = 1; + unsigned m = 1; + do + { + REV_BIT_CHECK(prob, i, m); + } + while (--numDirectBits); + } + } + } } -} -} - } - NORMALIZE_CHECK; - return res; + break; + } + NORMALIZE_CHECK; + + *bufOut = buf; + return res; } -static void LzmaDec_InitRc(CLzmaDec *p, const Byte *data) +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState); +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState) { - p->code = ((UInt32)data[1] << 24) | ((UInt32)data[2] << 16) | ((UInt32)data[3] << 8) | ((UInt32)data[4]); - p->range = 0xFFFFFFFF; - p->needFlush = 0; -} + p->remainLen = kMatchSpecLenStart + 1; + p->tempBufSize = 0; -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState) -{ - p->needFlush = 1; - p->remainLen = 0; - p->tempBufSize = 0; - - if (initDic) - { - p->processedPos = 0; - p->checkDicSize = 0; - p->needInitState = 1; - } - if (initState) - p->needInitState = 1; + if (initDic) + { + p->processedPos = 0; + p->checkDicSize = 0; + p->remainLen = kMatchSpecLenStart + 2; + } + if (initState) + p->remainLen = kMatchSpecLenStart + 2; } void LzmaDec_Init(CLzmaDec *p) { - p->dicPos = 0; - LzmaDec_InitDicAndState(p, True, True); + p->dicPos = 0; + LzmaDec_InitDicAndState(p, True, True); } -static void LzmaDec_InitStateReal(CLzmaDec *p) -{ - UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (p->prop.lc + p->prop.lp)); - UInt32 i; - CLzmaProb *probs = p->probs; - for (i = 0; i < numProbs; i++) - probs[i] = kBitModelTotal >> 1; - p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; - p->state = 0; - p->needInitState = 0; -} + +/* +LZMA supports optional end_marker. +So the decoder can lookahead for one additional LZMA-Symbol to check end_marker. +That additional LZMA-Symbol can require up to LZMA_REQUIRED_INPUT_MAX bytes in input stream. +When the decoder reaches dicLimit, it looks (finishMode) parameter: + if (finishMode == LZMA_FINISH_ANY), the decoder doesn't lookahead + if (finishMode != LZMA_FINISH_ANY), the decoder lookahead, if end_marker is possible for current position + +When the decoder lookahead, and the lookahead symbol is not end_marker, we have two ways: + 1) Strict mode (default) : the decoder returns SZ_ERROR_DATA. + 2) The relaxed mode (alternative mode) : we could return SZ_OK, and the caller + must check (status) value. The caller can show the error, + if the end of stream is expected, and the (status) is noit + LZMA_STATUS_FINISHED_WITH_MARK or LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK. +*/ + + +#define RETURN__NOT_FINISHED__FOR_FINISH \ + *status = LZMA_STATUS_NOT_FINISHED; \ + return SZ_ERROR_DATA; // for strict mode + // return SZ_OK; // for relaxed mode + SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { - SizeT inSize = *srcLen; - (*srcLen) = 0; + SizeT inSize = *srcLen; + (*srcLen) = 0; + *status = LZMA_STATUS_NOT_SPECIFIED; + + if (p->remainLen > kMatchSpecLenStart) + { + if (p->remainLen > kMatchSpecLenStart + 2) + return p->remainLen == kMatchSpecLen_Error_Fail ? SZ_ERROR_FAIL : SZ_ERROR_DATA; + + for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) + p->tempBuf[p->tempBufSize++] = *src++; + if (p->tempBufSize != 0 && p->tempBuf[0] != 0) + return SZ_ERROR_DATA; + if (p->tempBufSize < RC_INIT_SIZE) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + p->code = + ((UInt32)p->tempBuf[1] << 24) + | ((UInt32)p->tempBuf[2] << 16) + | ((UInt32)p->tempBuf[3] << 8) + | ((UInt32)p->tempBuf[4]); + + if (p->checkDicSize == 0 + && p->processedPos == 0 + && p->code >= kBadRepCode) + return SZ_ERROR_DATA; + + p->range = 0xFFFFFFFF; + p->tempBufSize = 0; + + if (p->remainLen > kMatchSpecLenStart + 1) + { + SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); + SizeT i; + CLzmaProb *probs = p->probs; + for (i = 0; i < numProbs; i++) + probs[i] = kBitModelTotal >> 1; + p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; + p->state = 0; + } + + p->remainLen = 0; + } + + for (;;) + { + if (p->remainLen == kMatchSpecLenStart) + { + if (p->code != 0) + return SZ_ERROR_DATA; + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } + LzmaDec_WriteRem(p, dicLimit); - *status = LZMA_STATUS_NOT_SPECIFIED; - - while (p->remainLen != kMatchSpecLenStart) { - int checkEndMarkNow; + // (p->remainLen == 0 || p->dicPos == dicLimit) - if (p->needFlush != 0) + int checkEndMarkNow = 0; + + if (p->dicPos >= dicLimit) + { + if (p->remainLen == 0 && p->code == 0) { - for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) - p->tempBuf[p->tempBufSize++] = *src++; - if (p->tempBufSize < RC_INIT_SIZE) - { - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (p->tempBuf[0] != 0) - return SZ_ERROR_DATA; - - LzmaDec_InitRc(p, p->tempBuf); - p->tempBufSize = 0; + *status = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK; + return SZ_OK; } - - checkEndMarkNow = 0; - if (p->dicPos >= dicLimit) + if (finishMode == LZMA_FINISH_ANY) { - if (p->remainLen == 0 && p->code == 0) - { - *status = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK; - return SZ_OK; - } - if (finishMode == LZMA_FINISH_ANY) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_OK; - } - if (p->remainLen != 0) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; - } - checkEndMarkNow = 1; + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_OK; } - - if (p->needInitState) - LzmaDec_InitStateReal(p); - - if (p->tempBufSize == 0) + if (p->remainLen != 0) { - SizeT processed; - const Byte *bufLimit; - if (inSize < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) - { - int dummyRes = LzmaDec_TryDummy(p, src, inSize); - if (dummyRes == DUMMY_ERROR) - { - memcpy(p->tempBuf, src, inSize); - p->tempBufSize = (unsigned)inSize; - (*srcLen) += inSize; - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (checkEndMarkNow && dummyRes != DUMMY_MATCH) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; - } - bufLimit = src; - } - else - bufLimit = src + inSize - LZMA_REQUIRED_INPUT_MAX; - p->buf = src; - if (LzmaDec_DecodeReal2(p, dicLimit, bufLimit) != 0) - return SZ_ERROR_DATA; - processed = (SizeT)(p->buf - src); - (*srcLen) += processed; - src += processed; - inSize -= processed; + RETURN__NOT_FINISHED__FOR_FINISH; + } + checkEndMarkNow = 1; + } + + // (p->remainLen == 0) + + if (p->tempBufSize == 0) + { + const Byte *bufLimit; + int dummyProcessed = -1; + + if (inSize < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) + { + const Byte *bufOut = src + inSize; + + ELzmaDummy dummyRes = LzmaDec_TryDummy(p, src, &bufOut); + + if (dummyRes == DUMMY_INPUT_EOF) + { + size_t i; + if (inSize >= LZMA_REQUIRED_INPUT_MAX) + break; + (*srcLen) += inSize; + p->tempBufSize = (unsigned)inSize; + for (i = 0; i < inSize; i++) + p->tempBuf[i] = src[i]; + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + + dummyProcessed = (int)(bufOut - src); + if ((unsigned)dummyProcessed > LZMA_REQUIRED_INPUT_MAX) + break; + + if (checkEndMarkNow && !IS_DUMMY_END_MARKER_POSSIBLE(dummyRes)) + { + unsigned i; + (*srcLen) += (unsigned)dummyProcessed; + p->tempBufSize = (unsigned)dummyProcessed; + for (i = 0; i < (unsigned)dummyProcessed; i++) + p->tempBuf[i] = src[i]; + // p->remainLen = kMatchSpecLen_Error_Data; + RETURN__NOT_FINISHED__FOR_FINISH; + } + + bufLimit = src; + // we will decode only one iteration } else + bufLimit = src + inSize - LZMA_REQUIRED_INPUT_MAX; + + p->buf = src; + { - unsigned rem = p->tempBufSize, lookAhead = 0; - while (rem < LZMA_REQUIRED_INPUT_MAX && lookAhead < inSize) - p->tempBuf[rem++] = src[lookAhead++]; - p->tempBufSize = rem; - if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) - { - int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, rem); - if (dummyRes == DUMMY_ERROR) - { - (*srcLen) += lookAhead; - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (checkEndMarkNow && dummyRes != DUMMY_MATCH) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; - } - } - p->buf = p->tempBuf; - if (LzmaDec_DecodeReal2(p, dicLimit, p->buf) != 0) - return SZ_ERROR_DATA; - lookAhead -= (rem - (unsigned)(p->buf - p->tempBuf)); - (*srcLen) += lookAhead; - src += lookAhead; - inSize -= lookAhead; - p->tempBufSize = 0; + int res = LzmaDec_DecodeReal2(p, dicLimit, bufLimit); + + SizeT processed = (SizeT)(p->buf - src); + + if (dummyProcessed < 0) + { + if (processed > inSize) + break; + } + else if ((unsigned)dummyProcessed != processed) + break; + + src += processed; + inSize -= processed; + (*srcLen) += processed; + + if (res != SZ_OK) + { + p->remainLen = kMatchSpecLen_Error_Data; + return SZ_ERROR_DATA; + } } + continue; + } + + { + // we have some data in (p->tempBuf) + // in strict mode: tempBufSize is not enough for one Symbol decoding. + // in relaxed mode: tempBufSize not larger than required for one Symbol decoding. + + unsigned rem = p->tempBufSize; + unsigned ahead = 0; + int dummyProcessed = -1; + + while (rem < LZMA_REQUIRED_INPUT_MAX && ahead < inSize) + p->tempBuf[rem++] = src[ahead++]; + + // ahead - the size of new data copied from (src) to (p->tempBuf) + // rem - the size of temp buffer including new data from (src) + + if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) + { + const Byte *bufOut = p->tempBuf + rem; + + ELzmaDummy dummyRes = LzmaDec_TryDummy(p, p->tempBuf, &bufOut); + + if (dummyRes == DUMMY_INPUT_EOF) + { + if (rem >= LZMA_REQUIRED_INPUT_MAX) + break; + p->tempBufSize = rem; + (*srcLen) += (SizeT)ahead; + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + + dummyProcessed = (int)(bufOut - p->tempBuf); + + if ((unsigned)dummyProcessed < p->tempBufSize) + break; + + if (checkEndMarkNow && !IS_DUMMY_END_MARKER_POSSIBLE(dummyRes)) + { + (*srcLen) += (unsigned)dummyProcessed - p->tempBufSize; + p->tempBufSize = (unsigned)dummyProcessed; + // p->remainLen = kMatchSpecLen_Error_Data; + RETURN__NOT_FINISHED__FOR_FINISH; + } + } + + p->buf = p->tempBuf; + + { + // we decode one symbol from (p->tempBuf) here, so the (bufLimit) is equal to (p->buf) + int res = LzmaDec_DecodeReal2(p, dicLimit, p->buf); + + SizeT processed = (SizeT)(p->buf - p->tempBuf); + rem = p->tempBufSize; + + if (dummyProcessed < 0) + { + if (processed > LZMA_REQUIRED_INPUT_MAX) + break; + if (processed < rem) + break; + } + else if ((unsigned)dummyProcessed != processed) + break; + + processed -= rem; + + src += processed; + inSize -= processed; + (*srcLen) += processed; + p->tempBufSize = 0; + + if (res != SZ_OK) + { + p->remainLen = kMatchSpecLen_Error_Data; + return SZ_ERROR_DATA; + } + } + } } - if (p->code == 0) - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA; + } + + /* Some unexpected error: internal error of code, memory corruption or hardware failure */ + p->remainLen = kMatchSpecLen_Error_Fail; + return SZ_ERROR_FAIL; } + + SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { - SizeT outSize = *destLen; - SizeT inSize = *srcLen; - *srcLen = *destLen = 0; - for (;;) + SizeT outSize = *destLen; + SizeT inSize = *srcLen; + *srcLen = *destLen = 0; + for (;;) + { + SizeT inSizeCur = inSize, outSizeCur, dicPos; + ELzmaFinishMode curFinishMode; + SRes res; + if (p->dicPos == p->dicBufSize) + p->dicPos = 0; + dicPos = p->dicPos; + if (outSize > p->dicBufSize - dicPos) { - SizeT inSizeCur = inSize, outSizeCur, dicPos; - ELzmaFinishMode curFinishMode; - SRes res; - if (p->dicPos == p->dicBufSize) - p->dicPos = 0; - dicPos = p->dicPos; - if (outSize > p->dicBufSize - dicPos) - { - outSizeCur = p->dicBufSize; - curFinishMode = LZMA_FINISH_ANY; - } - else - { - outSizeCur = dicPos + outSize; - curFinishMode = finishMode; - } - - res = LzmaDec_DecodeToDic(p, outSizeCur, src, &inSizeCur, curFinishMode, status); - src += inSizeCur; - inSize -= inSizeCur; - *srcLen += inSizeCur; - outSizeCur = p->dicPos - dicPos; - memcpy(dest, p->dic + dicPos, outSizeCur); - dest += outSizeCur; - outSize -= outSizeCur; - *destLen += outSizeCur; - if (res != 0) - return res; - if (outSizeCur == 0 || outSize == 0) - return SZ_OK; + outSizeCur = p->dicBufSize; + curFinishMode = LZMA_FINISH_ANY; } + else + { + outSizeCur = dicPos + outSize; + curFinishMode = finishMode; + } + + res = LzmaDec_DecodeToDic(p, outSizeCur, src, &inSizeCur, curFinishMode, status); + src += inSizeCur; + inSize -= inSizeCur; + *srcLen += inSizeCur; + outSizeCur = p->dicPos - dicPos; + memcpy(dest, p->dic + dicPos, outSizeCur); + dest += outSizeCur; + outSize -= outSizeCur; + *destLen += outSizeCur; + if (res != 0) + return res; + if (outSizeCur == 0 || outSize == 0) + return SZ_OK; + } } -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc) +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->probs); - p->probs = 0; + ISzAlloc_Free(alloc, p->probs); + p->probs = NULL; } -static void LzmaDec_FreeDict(CLzmaDec *p, ISzAlloc *alloc) +static void LzmaDec_FreeDict(CLzmaDec *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->dic); - p->dic = 0; + ISzAlloc_Free(alloc, p->dic); + p->dic = NULL; } -void LzmaDec_Free(CLzmaDec *p, ISzAlloc *alloc) +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc) { - LzmaDec_FreeProbs(p, alloc); - LzmaDec_FreeDict(p, alloc); + LzmaDec_FreeProbs(p, alloc); + LzmaDec_FreeDict(p, alloc); } SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) { - UInt32 dicSize; - Byte d; + UInt32 dicSize; + Byte d; + + if (size < LZMA_PROPS_SIZE) + return SZ_ERROR_UNSUPPORTED; + else + dicSize = data[1] | ((UInt32)data[2] << 8) | ((UInt32)data[3] << 16) | ((UInt32)data[4] << 24); + + if (dicSize < LZMA_DIC_MIN) + dicSize = LZMA_DIC_MIN; + p->dicSize = dicSize; - if (size < LZMA_PROPS_SIZE) - return SZ_ERROR_UNSUPPORTED; - else - dicSize = data[1] | ((UInt32)data[2] << 8) | ((UInt32)data[3] << 16) | ((UInt32)data[4] << 24); + d = data[0]; + if (d >= (9 * 5 * 5)) + return SZ_ERROR_UNSUPPORTED; - if (dicSize < LZMA_DIC_MIN) - dicSize = LZMA_DIC_MIN; - p->dicSize = dicSize; + p->lc = (Byte)(d % 9); + d /= 9; + p->pb = (Byte)(d / 5); + p->lp = (Byte)(d % 5); - d = data[0]; - if (d >= (9 * 5 * 5)) - return SZ_ERROR_UNSUPPORTED; - - p->lc = d % 9; - d /= 9; - p->pb = d / 5; - p->lp = d % 5; - - return SZ_OK; + return SZ_OK; } -static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAlloc *alloc) +static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAllocPtr alloc) { - UInt32 numProbs = LzmaProps_GetNumProbs(propNew); - if (p->probs == 0 || numProbs != p->numProbs) + UInt32 numProbs = LzmaProps_GetNumProbs(propNew); + if (!p->probs || numProbs != p->numProbs) + { + LzmaDec_FreeProbs(p, alloc); + p->probs = (CLzmaProb *)ISzAlloc_Alloc(alloc, numProbs * sizeof(CLzmaProb)); + if (!p->probs) + return SZ_ERROR_MEM; + p->probs_1664 = p->probs + 1664; + p->numProbs = numProbs; + } + return SZ_OK; +} + +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) +{ + CLzmaProps propNew; + RINOK(LzmaProps_Decode(&propNew, props, propsSize)); + RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); + p->prop = propNew; + return SZ_OK; +} + +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) +{ + CLzmaProps propNew; + SizeT dicBufSize; + RINOK(LzmaProps_Decode(&propNew, props, propsSize)); + RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); + + { + UInt32 dictSize = propNew.dicSize; + SizeT mask = ((UInt32)1 << 12) - 1; + if (dictSize >= ((UInt32)1 << 30)) mask = ((UInt32)1 << 22) - 1; + else if (dictSize >= ((UInt32)1 << 22)) mask = ((UInt32)1 << 20) - 1;; + dicBufSize = ((SizeT)dictSize + mask) & ~mask; + if (dicBufSize < dictSize) + dicBufSize = dictSize; + } + + if (!p->dic || dicBufSize != p->dicBufSize) + { + LzmaDec_FreeDict(p, alloc); + p->dic = (Byte *)ISzAlloc_Alloc(alloc, dicBufSize); + if (!p->dic) { - LzmaDec_FreeProbs(p, alloc); - p->probs = (CLzmaProb *)alloc->Alloc(alloc, numProbs * sizeof(CLzmaProb)); - p->numProbs = numProbs; - if (p->probs == 0) - return SZ_ERROR_MEM; + LzmaDec_FreeProbs(p, alloc); + return SZ_ERROR_MEM; } - return SZ_OK; -} - -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) -{ - CLzmaProps propNew; - RINOK(LzmaProps_Decode(&propNew, props, propsSize)); - RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); - p->prop = propNew; - return SZ_OK; -} - -SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) -{ - CLzmaProps propNew; - SizeT dicBufSize; - RINOK(LzmaProps_Decode(&propNew, props, propsSize)); - RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); - dicBufSize = propNew.dicSize; - if (p->dic == 0 || dicBufSize != p->dicBufSize) - { - LzmaDec_FreeDict(p, alloc); - p->dic = (Byte *)alloc->Alloc(alloc, dicBufSize); - if (p->dic == 0) - { - LzmaDec_FreeProbs(p, alloc); - return SZ_ERROR_MEM; - } - } - p->dicBufSize = dicBufSize; - p->prop = propNew; - return SZ_OK; + } + p->dicBufSize = dicBufSize; + p->prop = propNew; + return SZ_OK; } SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAlloc *alloc) + ELzmaStatus *status, ISzAllocPtr alloc) { - CLzmaDec p; - SRes res; - SizeT inSize = *srcLen; - SizeT outSize = *destLen; - *srcLen = *destLen = 0; - if (inSize < RC_INIT_SIZE) - return SZ_ERROR_INPUT_EOF; - - LzmaDec_Construct(&p); - res = LzmaDec_AllocateProbs(&p, propData, propSize, alloc); - if (res != 0) - return res; - p.dic = dest; - p.dicBufSize = outSize; - - LzmaDec_Init(&p); - - *srcLen = inSize; - res = LzmaDec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status); - - if (res == SZ_OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT) - res = SZ_ERROR_INPUT_EOF; - - (*destLen) = p.dicPos; - LzmaDec_FreeProbs(&p, alloc); - return res; -} \ No newline at end of file + CLzmaDec p; + SRes res; + SizeT outSize = *destLen, inSize = *srcLen; + *destLen = *srcLen = 0; + *status = LZMA_STATUS_NOT_SPECIFIED; + if (inSize < RC_INIT_SIZE) + return SZ_ERROR_INPUT_EOF; + LzmaDec_Construct(&p); + RINOK(LzmaDec_AllocateProbs(&p, propData, propSize, alloc)); + p.dic = dest; + p.dicBufSize = outSize; + LzmaDec_Init(&p); + *srcLen = inSize; + res = LzmaDec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status); + *destLen = p.dicPos; + if (res == SZ_OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT) + res = SZ_ERROR_INPUT_EOF; + LzmaDec_FreeProbs(&p, alloc); + return res; +} diff --git a/common/LZMA/SDK/C/LzmaDec.h b/common/LZMA/SDK/C/LzmaDec.h index bf7f084..6f12962 100644 --- a/common/LZMA/SDK/C/LzmaDec.h +++ b/common/LZMA/SDK/C/LzmaDec.h @@ -1,24 +1,24 @@ /* LzmaDec.h -- LZMA Decoder -2009-02-07 : Igor Pavlov : Public domain */ +2020-03-19 : Igor Pavlov : Public domain */ #ifndef __LZMA_DEC_H #define __LZMA_DEC_H -#include "Types.h" +#include "7zTypes.h" -#ifdef __cplusplus -extern "C" { -#endif +EXTERN_C_BEGIN /* #define _LZMA_PROB32 */ /* _LZMA_PROB32 can increase the speed on some CPUs, but memory usage for CLzmaDec::probs will be doubled in that case */ +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; /* ---------- LZMA Properties ---------- */ @@ -27,7 +27,10 @@ extern "C" { typedef struct _CLzmaProps { - unsigned lc, lp, pb; + Byte lc; + Byte lp; + Byte pb; + Byte _pad_; UInt32 dicSize; } CLzmaProps; @@ -49,32 +52,34 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size); typedef struct { + /* Don't change this structure. ASM code can use it. */ CLzmaProps prop; CLzmaProb *probs; + CLzmaProb *probs_1664; Byte *dic; - const Byte *buf; - UInt32 range, code; - SizeT dicPos; SizeT dicBufSize; + SizeT dicPos; + const Byte *buf; + UInt32 range; + UInt32 code; UInt32 processedPos; UInt32 checkDicSize; - unsigned state; UInt32 reps[4]; - unsigned remainLen; - int needFlush; - int needInitState; + UInt32 state; + UInt32 remainLen; + UInt32 numProbs; unsigned tempBufSize; Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; } CLzmaDec; -#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; } +#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; } void LzmaDec_Init(CLzmaDec *p); /* There are two types of LZMA streams: - 0) Stream with end mark. That end mark adds about 6 bytes to compressed size. - 1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */ + - Stream with end mark. That end mark adds about 6 bytes to compressed size. + - Stream without end mark. You must know exact uncompressed size to decompress such stream. */ typedef enum { @@ -131,11 +136,11 @@ LzmaDec_Allocate* can return: SZ_ERROR_UNSUPPORTED - Unsupported properties */ -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc); -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc); +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc); -SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAlloc *alloc); -void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc); +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc); /* ---------- Dictionary Interface ---------- */ @@ -144,7 +149,7 @@ void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc); You must work with CLzmaDec variables directly in this interface. STEPS: - LzmaDec_Constr() + LzmaDec_Construct() LzmaDec_Allocate() for (each new stream) { @@ -176,6 +181,7 @@ Returns: LZMA_STATUS_NEEDS_MORE_INPUT LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK SZ_ERROR_DATA - Data error + SZ_ERROR_FAIL - Some unexpected error: internal error of code, memory corruption or hardware failure */ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, @@ -218,14 +224,13 @@ Returns: SZ_ERROR_MEM - Memory allocation error SZ_ERROR_UNSUPPORTED - Unsupported properties SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). + SZ_ERROR_FAIL - Some unexpected error: internal error of code, memory corruption or hardware failure */ SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAlloc *alloc); + ELzmaStatus *status, ISzAllocPtr alloc); -#ifdef __cplusplus -} -#endif +EXTERN_C_END #endif diff --git a/common/LZMA/SDK/C/LzmaEnc.c b/common/LZMA/SDK/C/LzmaEnc.c index 11fdfe0..c8b31a1 100644 --- a/common/LZMA/SDK/C/LzmaEnc.c +++ b/common/LZMA/SDK/C/LzmaEnc.c @@ -1,5 +1,7 @@ /* LzmaEnc.c -- LZMA Encoder -2010-04-16 : Igor Pavlov : Public domain*/ +2022-07-15: Igor Pavlov : Public domain */ + +#include "Precomp.h" #include @@ -10,6 +12,7 @@ #include #endif +#include "CpuArch.h" #include "LzmaEnc.h" #include "LzFind.h" @@ -17,18 +20,25 @@ #include "LzFindMt.h" #endif +/* the following LzmaEnc_* declarations is internal LZMA interface for LZMA2 encoder */ + +SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, + ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, + Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); +const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); +void LzmaEnc_Finish(CLzmaEncHandle pp); +void LzmaEnc_SaveState(CLzmaEncHandle pp); +void LzmaEnc_RestoreState(CLzmaEncHandle pp); + #ifdef SHOW_STAT -static int ttt = 0; +static unsigned g_STAT_OFFSET = 0; #endif -#define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1) - -#define kBlockSize (9 << 10) -#define kUnpackBlockSize (1 << 18) -#define kMatchArraySize (1 << 21) -#define kMatchRecordMaxSize ((LZMA_MATCH_LEN_MAX * 2 + 3) * LZMA_MATCH_LEN_MAX) - -#define kNumMaxDirectBits (31) +/* for good normalization speed we still reserve 256 MB before 4 GB range */ +#define kLzmaMaxHistorySize ((UInt32)15 << 28) #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits) @@ -40,124 +50,237 @@ static int ttt = 0; #define kNumMoveReducingBits 4 #define kNumBitPriceShiftBits 4 -#define kBitPrice (1 << kNumBitPriceShiftBits) +// #define kBitPrice (1 << kNumBitPriceShiftBits) + +#define REP_LEN_COUNT 64 void LzmaEncProps_Init(CLzmaEncProps *p) { - p->level = 5; - p->dictSize = p->mc = 0; - p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1; - p->writeEndMark = 0; + p->level = 5; + p->dictSize = p->mc = 0; + p->reduceSize = (UInt64)(Int64)-1; + p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1; + p->writeEndMark = 0; + p->affinity = 0; } void LzmaEncProps_Normalize(CLzmaEncProps *p) { - int level = p->level; - if (level < 0) level = 5; - p->level = level; - if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26))); - if (p->lc < 0) p->lc = 3; - if (p->lp < 0) p->lp = 0; - if (p->pb < 0) p->pb = 2; - if (p->algo < 0) p->algo = (level < 5 ? 0 : 1); - if (p->fb < 0) p->fb = (level < 7 ? 32 : 64); - if (p->btMode < 0) p->btMode = (p->algo == 0 ? 0 : 1); - if (p->numHashBytes < 0) p->numHashBytes = 4; - if (p->mc == 0) p->mc = (16 + (p->fb >> 1)) >> (p->btMode ? 0 : 1); - if (p->numThreads < 0) - p->numThreads = -#ifndef _7ZIP_ST - ((p->btMode && p->algo) ? 2 : 1); -#else - 1; -#endif + int level = p->level; + if (level < 0) level = 5; + p->level = level; + + if (p->dictSize == 0) + p->dictSize = + ( level <= 3 ? ((UInt32)1 << (level * 2 + 16)) : + ( level <= 6 ? ((UInt32)1 << (level + 19)) : + ( level <= 7 ? ((UInt32)1 << 25) : ((UInt32)1 << 26) + ))); + + if (p->dictSize > p->reduceSize) + { + UInt32 v = (UInt32)p->reduceSize; + const UInt32 kReduceMin = ((UInt32)1 << 12); + if (v < kReduceMin) + v = kReduceMin; + if (p->dictSize > v) + p->dictSize = v; + } + + if (p->lc < 0) p->lc = 3; + if (p->lp < 0) p->lp = 0; + if (p->pb < 0) p->pb = 2; + + if (p->algo < 0) p->algo = (level < 5 ? 0 : 1); + if (p->fb < 0) p->fb = (level < 7 ? 32 : 64); + if (p->btMode < 0) p->btMode = (p->algo == 0 ? 0 : 1); + if (p->numHashBytes < 0) p->numHashBytes = (p->btMode ? 4 : 5); + if (p->mc == 0) p->mc = (16 + ((unsigned)p->fb >> 1)) >> (p->btMode ? 0 : 1); + + if (p->numThreads < 0) + p->numThreads = + #ifndef _7ZIP_ST + ((p->btMode && p->algo) ? 2 : 1); + #else + 1; + #endif } UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2) { - CLzmaEncProps props = *props2; - LzmaEncProps_Normalize(&props); - return props.dictSize; + CLzmaEncProps props = *props2; + LzmaEncProps_Normalize(&props); + return props.dictSize; } -/* #define LZMA_LOG_BSR */ -/* Define it for Intel's CPU */ + +/* +x86/x64: + +BSR: + IF (SRC == 0) ZF = 1, DEST is undefined; + AMD : DEST is unchanged; + IF (SRC != 0) ZF = 0; DEST is index of top non-zero bit + BSR is slow in some processors + +LZCNT: + IF (SRC == 0) CF = 1, DEST is size_in_bits_of_register(src) (32 or 64) + IF (SRC != 0) CF = 0, DEST = num_lead_zero_bits + IF (DEST == 0) ZF = 1; + +LZCNT works only in new processors starting from Haswell. +if LZCNT is not supported by processor, then it's executed as BSR. +LZCNT can be faster than BSR, if supported. +*/ + +// #define LZMA_LOG_BSR + +#if defined(MY_CPU_ARM_OR_ARM64) /* || defined(MY_CPU_X86_OR_AMD64) */ + + #if (defined(__clang__) && (__clang_major__ >= 6)) \ + || (defined(__GNUC__) && (__GNUC__ >= 6)) + #define LZMA_LOG_BSR + #elif defined(_MSC_VER) && (_MSC_VER >= 1300) + // #if defined(MY_CPU_ARM_OR_ARM64) + #define LZMA_LOG_BSR + // #endif + #endif +#endif + +// #include #ifdef LZMA_LOG_BSR -#define kDicLogSizeMaxCompress 30 +#if defined(__clang__) \ + || defined(__GNUC__) -#define BSR2_RET(pos, res) { unsigned long i; _BitScanReverse(&i, (pos)); res = (i + i) + ((pos >> (i - 1)) & 1); } +/* + C code: : (30 - __builtin_clz(x)) + gcc9/gcc10 for x64 /x86 : 30 - (bsr(x) xor 31) + clang10 for x64 : 31 + (bsr(x) xor -32) +*/ -UInt32 GetPosSlot1(UInt32 pos) + #define MY_clz(x) ((unsigned)__builtin_clz(x)) + // __lzcnt32 + // __builtin_ia32_lzcnt_u32 + +#else // #if defined(_MSC_VER) + + #ifdef MY_CPU_ARM_OR_ARM64 + + #define MY_clz _CountLeadingZeros + + #else // if defined(MY_CPU_X86_OR_AMD64) + + // #define MY_clz __lzcnt // we can use lzcnt (unsupported by old CPU) + // _BitScanReverse code is not optimal for some MSVC compilers + #define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); zz--; \ + res = (zz + zz) + (pos >> zz); } + + #endif // MY_CPU_X86_OR_AMD64 + +#endif // _MSC_VER + + +#ifndef BSR2_RET + + #define BSR2_RET(pos, res) { unsigned zz = 30 - MY_clz(pos); \ + res = (zz + zz) + (pos >> zz); } + +#endif + + +unsigned GetPosSlot1(UInt32 pos); +unsigned GetPosSlot1(UInt32 pos) { - UInt32 res; - BSR2_RET(pos, res); - return res; + unsigned res; + BSR2_RET(pos, res); + return res; } #define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } #define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res); } -#else -#define kNumLogBits (9 + (int)sizeof(size_t) / 2) +#else // ! LZMA_LOG_BSR + +#define kNumLogBits (11 + sizeof(size_t) / 8 * 3) + #define kDicLogSizeMaxCompress ((kNumLogBits - 1) * 2 + 7) -void LzmaEnc_FastPosInit(Byte *g_FastPos) +static void LzmaEnc_FastPosInit(Byte *g_FastPos) { - int c = 2, slotFast; - g_FastPos[0] = 0; - g_FastPos[1] = 1; - - for (slotFast = 2; slotFast < kNumLogBits * 2; slotFast++) - { - UInt32 k = (1 << ((slotFast >> 1) - 1)); - UInt32 j; - for (j = 0; j < k; j++, c++) - g_FastPos[c] = (Byte)slotFast; - } + unsigned slot; + g_FastPos[0] = 0; + g_FastPos[1] = 1; + g_FastPos += 2; + + for (slot = 2; slot < kNumLogBits * 2; slot++) + { + size_t k = ((size_t)1 << ((slot >> 1) - 1)); + size_t j; + for (j = 0; j < k; j++) + g_FastPos[j] = (Byte)slot; + g_FastPos += k; + } } -#define BSR2_RET(pos, res) { UInt32 i = 6 + ((kNumLogBits - 1) & \ +/* we can use ((limit - pos) >> 31) only if (pos < ((UInt32)1 << 31)) */ +/* +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ (0 - (((((UInt32)1 << (kNumLogBits + 6)) - 1) - pos) >> 31))); \ - res = p->g_FastPos[pos >> i] + (i * 2); } + res = p->g_FastPos[pos >> zz] + (zz * 2); } +*/ + +/* +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ + (0 - (((((UInt32)1 << (kNumLogBits)) - 1) - (pos >> 6)) >> 31))); \ + res = p->g_FastPos[pos >> zz] + (zz * 2); } +*/ + +#define BSR2_RET(pos, res) { unsigned zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ + res = p->g_FastPos[pos >> zz] + (zz * 2); } + /* #define BSR2_RET(pos, res) { res = (pos < (1 << (kNumLogBits + 6))) ? \ p->g_FastPos[pos >> 6] + 12 : \ p->g_FastPos[pos >> (6 + kNumLogBits - 1)] + (6 + (kNumLogBits - 1)) * 2; } - */ +*/ #define GetPosSlot1(pos) p->g_FastPos[pos] #define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos]; else BSR2_RET(pos, res); } +#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos & (kNumFullDistances - 1)]; else BSR2_RET(pos, res); } + +#endif // LZMA_LOG_BSR -#endif #define LZMA_NUM_REPS 4 -typedef unsigned CState; +typedef UInt16 CState; +typedef UInt16 CExtra; typedef struct { - UInt32 price; - - CState state; - int prev1IsChar; - int prev2; - - UInt32 posPrev2; - UInt32 backPrev2; - - UInt32 posPrev; - UInt32 backPrev; - UInt32 backs[LZMA_NUM_REPS]; + UInt32 price; + CState state; + CExtra extra; + // 0 : normal + // 1 : LIT : MATCH + // > 1 : MATCH (extra-1) : LIT : REP0 (len) + UInt32 len; + UInt32 dist; + UInt32 reps[LZMA_NUM_REPS]; } COptimal; -#define kNumOpts (1 << 12) + +// 18.06 +#define kNumOpts (1 << 11) +#define kPackReserve (kNumOpts * 8) +// #define kNumOpts (1 << 12) +// #define kPackReserve (1 + kNumOpts * 2) #define kNumLenToPosStates 4 #define kNumPosSlotBits 6 -#define kDicLogSizeM0 +// #define kDicLogSizeMin 0 #define kDicLogSizeMax 32 #define kDistTableSizeMax (kDicLogSizeMax * 2) @@ -167,15 +290,15 @@ typedef struct #define kStartPosModelIndex 4 #define kEndPosModelIndex 14 -#define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex) - #define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; #define LZMA_PB_MAX 4 #define LZMA_LC_MAX 8 @@ -185,2064 +308,2855 @@ typedef struct #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define kLenNumSymbolsTotal (kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kLenNumSymbolsTotal (kLenNumLowSymbols * 2 + kLenNumHighSymbols) #define LZMA_MATCH_LEN_MIN 2 #define LZMA_MATCH_LEN_MAX (LZMA_MATCH_LEN_MIN + kLenNumSymbolsTotal - 1) #define kNumStates 12 + typedef struct { - CLzmaProb choice; - CLzmaProb choice2; - CLzmaProb low[LZMA_NUM_PB_STATES_MAX << kLenNumLowBits]; - CLzmaProb mid[LZMA_NUM_PB_STATES_MAX << kLenNumMidBits]; - CLzmaProb high[kLenNumHighSymbols]; + CLzmaProb low[LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)]; + CLzmaProb high[kLenNumHighSymbols]; } CLenEnc; + typedef struct { - CLenEnc p; - UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; - UInt32 tableSize; - UInt32 counters[LZMA_NUM_PB_STATES_MAX]; + unsigned tableSize; + UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; + // UInt32 prices1[LZMA_NUM_PB_STATES_MAX][kLenNumLowSymbols * 2]; + // UInt32 prices2[kLenNumSymbolsTotal]; } CLenPriceEnc; +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices[posState][(size_t)(len) - LZMA_MATCH_LEN_MIN]) + +/* +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices2[(size_t)(len) - 2] + ((p)->prices1[posState][((len) - 2) & (kLenNumLowSymbols * 2 - 1)] & (((len) - 2 - kLenNumLowSymbols * 2) >> 9))) +*/ + typedef struct { - UInt32 range; - Byte cache; - UInt64 low; - UInt64 cacheSize; - Byte *buf; - Byte *bufLim; - Byte *bufBase; - ISeqOutStream *outStream; - UInt64 processed; - SRes res; + UInt32 range; + unsigned cache; + UInt64 low; + UInt64 cacheSize; + Byte *buf; + Byte *bufLim; + Byte *bufBase; + ISeqOutStream *outStream; + UInt64 processed; + SRes res; } CRangeEnc; + typedef struct { - CLzmaProb *litProbs; + CLzmaProb *litProbs; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb isRep[kNumStates]; - CLzmaProb isRepG0[kNumStates]; - CLzmaProb isRepG1[kNumStates]; - CLzmaProb isRepG2[kNumStates]; - CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; + unsigned state; + UInt32 reps[LZMA_NUM_REPS]; - CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb isRep[kNumStates]; + CLzmaProb isRepG0[kNumStates]; + CLzmaProb isRepG1[kNumStates]; + CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLenPriceEnc lenEnc; - CLenPriceEnc repLenEnc; + CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; + CLzmaProb posEncoders[kNumFullDistances]; + + CLenEnc lenProbs; + CLenEnc repLenProbs; - UInt32 reps[LZMA_NUM_REPS]; - UInt32 state; } CSaveState; + +typedef UInt32 CProbPrice; + + typedef struct { - IMatchFinder matchFinder; - void *matchFinderObj; + void *matchFinderObj; + IMatchFinder2 matchFinder; -#ifndef _7ZIP_ST - Bool mtMode; - CMatchFinderMt matchFinderMt; -#endif + unsigned optCur; + unsigned optEnd; - CMatchFinder matchFinderBase; + unsigned longestMatchLen; + unsigned numPairs; + UInt32 numAvail; -#ifndef _7ZIP_ST - Byte pad[128]; -#endif + unsigned state; + unsigned numFastBytes; + unsigned additionalOffset; + UInt32 reps[LZMA_NUM_REPS]; + unsigned lpMask, pbMask; + CLzmaProb *litProbs; + CRangeEnc rc; - UInt32 optimumEndIndex; - UInt32 optimumCurrentIndex; + UInt32 backRes; - UInt32 longestMatchLength; - UInt32 numPairs; - UInt32 numAvail; - COptimal opt[kNumOpts]; + unsigned lc, lp, pb; + unsigned lclp; -#ifndef LZMA_LOG_BSR - Byte g_FastPos[1 << kNumLogBits]; -#endif + BoolInt fastMode; + BoolInt writeEndMark; + BoolInt finished; + BoolInt multiThread; + BoolInt needInit; + // BoolInt _maxMode; - UInt32 ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; - UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1]; - UInt32 numFastBytes; - UInt32 additionalOffset; - UInt32 reps[LZMA_NUM_REPS]; - UInt32 state; + UInt64 nowPos64; + + unsigned matchPriceCount; + // unsigned alignPriceCount; + int repLenEncCounter; - UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; - UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; - UInt32 alignPrices[kAlignTableSize]; - UInt32 alignPriceCount; + unsigned distTableSize; - UInt32 distTableSize; + UInt32 dictSize; + SRes result; - unsigned lc, lp, pb; - unsigned lpMask, pbMask; + #ifndef _7ZIP_ST + BoolInt mtMode; + // begin of CMatchFinderMt is used in LZ thread + CMatchFinderMt matchFinderMt; + // end of CMatchFinderMt is used in BT and HASH threads + // #else + // CMatchFinder matchFinderBase; + #endif + CMatchFinder matchFinderBase; - CLzmaProb *litProbs; + + // we suppose that we have 8-bytes alignment after CMatchFinder + + #ifndef _7ZIP_ST + Byte pad[128]; + #endif + + // LZ thread + CProbPrice ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb isRep[kNumStates]; - CLzmaProb isRepG0[kNumStates]; - CLzmaProb isRepG1[kNumStates]; - CLzmaProb isRepG2[kNumStates]; - CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; + // we want {len , dist} pairs to be 8-bytes aligned in matches array + UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2]; - CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + // we want 8-bytes alignment here + UInt32 alignPrices[kAlignTableSize]; + UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; + UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; - CLenPriceEnc lenEnc; - CLenPriceEnc repLenEnc; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb isRep[kNumStates]; + CLzmaProb isRepG0[kNumStates]; + CLzmaProb isRepG1[kNumStates]; + CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; + CLzmaProb posEncoders[kNumFullDistances]; + + CLenEnc lenProbs; + CLenEnc repLenProbs; - unsigned lclp; + #ifndef LZMA_LOG_BSR + Byte g_FastPos[1 << kNumLogBits]; + #endif - Bool fastMode; + CLenPriceEnc lenEnc; + CLenPriceEnc repLenEnc; - CRangeEnc rc; + COptimal opt[kNumOpts]; - Bool writeEndMark; - UInt64 nowPos64; - UInt32 matchPriceCount; - Bool finished; - Bool multiThread; + CSaveState saveState; - SRes result; - UInt32 dictSize; - UInt32 matchFinderCycles; - - int needInit; - - CSaveState saveState; + // BoolInt mf_Failure; + #ifndef _7ZIP_ST + Byte pad2[128]; + #endif } CLzmaEnc; + +#define MFB (p->matchFinderBase) +/* +#ifndef _7ZIP_ST +#define MFB (p->matchFinderMt.MatchFinder) +#endif +*/ + +#define COPY_ARR(dest, src, arr) memcpy(dest->arr, src->arr, sizeof(src->arr)); + void LzmaEnc_SaveState(CLzmaEncHandle pp) { - CLzmaEnc *p = (CLzmaEnc *)pp; - CSaveState *dest = &p->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; - dest->state = p->state; + CLzmaEnc *p = (CLzmaEnc *)pp; + CSaveState *dest = &p->saveState; + + dest->state = p->state; + + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); - memcpy(dest->litProbs, p->litProbs, (0x300 << p->lclp) * sizeof(CLzmaProb)); + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); + + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << p->lclp) * sizeof(CLzmaProb)); } + void LzmaEnc_RestoreState(CLzmaEncHandle pp) { - CLzmaEnc *dest = (CLzmaEnc *)pp; - const CSaveState *p = &dest->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; - dest->state = p->state; + CLzmaEnc *dest = (CLzmaEnc *)pp; + const CSaveState *p = &dest->saveState; - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); - memcpy(dest->litProbs, p->litProbs, (0x300 << dest->lclp) * sizeof(CLzmaProb)); + dest->state = p->state; + + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); + + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << dest->lclp) * sizeof(CLzmaProb)); } + + SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) { - CLzmaEnc *p = (CLzmaEnc *)pp; - CLzmaEncProps props = *props2; - LzmaEncProps_Normalize(&props); + CLzmaEnc *p = (CLzmaEnc *)pp; + CLzmaEncProps props = *props2; + LzmaEncProps_Normalize(&props); - if (props.lc > LZMA_LC_MAX || props.lp > LZMA_LP_MAX || props.pb > LZMA_PB_MAX || - props.dictSize > ((UInt32)1 << kDicLogSizeMaxCompress) || props.dictSize > ((UInt32)1 << 30)) - return SZ_ERROR_PARAM; - p->dictSize = props.dictSize; - p->matchFinderCycles = props.mc; + if (props.lc > LZMA_LC_MAX + || props.lp > LZMA_LP_MAX + || props.pb > LZMA_PB_MAX) + return SZ_ERROR_PARAM; + + + if (props.dictSize > kLzmaMaxHistorySize) + props.dictSize = kLzmaMaxHistorySize; + + #ifndef LZMA_LOG_BSR + { + const UInt64 dict64 = props.dictSize; + if (dict64 > ((UInt64)1 << kDicLogSizeMaxCompress)) + return SZ_ERROR_PARAM; + } + #endif + + p->dictSize = props.dictSize; + { + unsigned fb = (unsigned)props.fb; + if (fb < 5) + fb = 5; + if (fb > LZMA_MATCH_LEN_MAX) + fb = LZMA_MATCH_LEN_MAX; + p->numFastBytes = fb; + } + p->lc = (unsigned)props.lc; + p->lp = (unsigned)props.lp; + p->pb = (unsigned)props.pb; + p->fastMode = (props.algo == 0); + // p->_maxMode = True; + MFB.btMode = (Byte)(props.btMode ? 1 : 0); + { + unsigned numHashBytes = 4; + if (props.btMode) { - unsigned fb = props.fb; - if (fb < 5) - fb = 5; - if (fb > LZMA_MATCH_LEN_MAX) - fb = LZMA_MATCH_LEN_MAX; - p->numFastBytes = fb; - } - p->lc = props.lc; - p->lp = props.lp; - p->pb = props.pb; - p->fastMode = (props.algo == 0); - p->matchFinderBase.btMode = props.btMode; - { - UInt32 numHashBytes = 4; - if (props.btMode) - { - if (props.numHashBytes < 2) - numHashBytes = 2; - else if (props.numHashBytes < 4) - numHashBytes = props.numHashBytes; - } - p->matchFinderBase.numHashBytes = numHashBytes; + if (props.numHashBytes < 2) numHashBytes = 2; + else if (props.numHashBytes < 4) numHashBytes = (unsigned)props.numHashBytes; } + if (props.numHashBytes >= 5) numHashBytes = 5; - p->matchFinderBase.cutValue = props.mc; + MFB.numHashBytes = numHashBytes; + } - p->writeEndMark = props.writeEndMark; + MFB.cutValue = props.mc; -#ifndef _7ZIP_ST - /* - if (newMultiThread != _multiThread) - { + p->writeEndMark = (BoolInt)props.writeEndMark; + + #ifndef _7ZIP_ST + /* + if (newMultiThread != _multiThread) + { ReleaseMatchFinder(); _multiThread = newMultiThread; - } - */ - p->multiThread = (props.numThreads > 1); -#endif + } + */ + p->multiThread = (props.numThreads > 1); + p->matchFinderMt.btSync.affinity = + p->matchFinderMt.hashSync.affinity = props.affinity; + #endif - return SZ_OK; + return SZ_OK; } -static const int kLiteralNextStates[kNumStates] = { 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5 }; -static const int kMatchNextStates[kNumStates] = { 7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10 }; -static const int kRepNextStates[kNumStates] = { 8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11 }; -static const int kShortRepNextStates[kNumStates] = { 9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11 }; -#define IsCharState(s) ((s) < 7) +void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + MFB.expectedDataSize = expectedDataSiize; +} + +#define kState_Start 0 +#define kState_LitAfterMatch 4 +#define kState_LitAfterRep 5 +#define kState_MatchAfterLit 7 +#define kState_RepAfterLit 8 + +static const Byte kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; +static const Byte kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; +static const Byte kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; +static const Byte kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; + +#define IsLitState(s) ((s) < 7) +#define GetLenToPosState2(len) (((len) < kNumLenToPosStates - 1) ? (len) : kNumLenToPosStates - 1) #define GetLenToPosState(len) (((len) < kNumLenToPosStates + 1) ? (len) - 2 : kNumLenToPosStates - 1) #define kInfinityPrice (1 << 30) static void RangeEnc_Construct(CRangeEnc *p) { - p->outStream = 0; - p->bufBase = 0; + p->outStream = NULL; + p->bufBase = NULL; } -#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed(p) ( (p)->processed + (size_t)((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed_sizet(p) ((size_t)(p)->processed + (size_t)((p)->buf - (p)->bufBase) + (size_t)(p)->cacheSize) #define RC_BUF_SIZE (1 << 16) -static int RangeEnc_Alloc(CRangeEnc *p, ISzAlloc *alloc) + +static int RangeEnc_Alloc(CRangeEnc *p, ISzAllocPtr alloc) { - if (p->bufBase == 0) - { - p->bufBase = (Byte *)alloc->Alloc(alloc, RC_BUF_SIZE); - if (p->bufBase == 0) - return 0; - p->bufLim = p->bufBase + RC_BUF_SIZE; - } - return 1; + if (!p->bufBase) + { + p->bufBase = (Byte *)ISzAlloc_Alloc(alloc, RC_BUF_SIZE); + if (!p->bufBase) + return 0; + p->bufLim = p->bufBase + RC_BUF_SIZE; + } + return 1; } -static void RangeEnc_Free(CRangeEnc *p, ISzAlloc *alloc) +static void RangeEnc_Free(CRangeEnc *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->bufBase); - p->bufBase = 0; + ISzAlloc_Free(alloc, p->bufBase); + p->bufBase = NULL; } static void RangeEnc_Init(CRangeEnc *p) { - /* Stream.Init(); */ - p->low = 0; - p->range = 0xFFFFFFFF; - p->cacheSize = 1; - p->cache = 0; + p->range = 0xFFFFFFFF; + p->cache = 0; + p->low = 0; + p->cacheSize = 0; - p->buf = p->bufBase; + p->buf = p->bufBase; - p->processed = 0; - p->res = SZ_OK; + p->processed = 0; + p->res = SZ_OK; } -static void RangeEnc_FlushStream(CRangeEnc *p) +MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) { - size_t num; - if (p->res != SZ_OK) - return; - num = p->buf - p->bufBase; - if (num != p->outStream->Write(p->outStream, p->bufBase, num)) - p->res = SZ_ERROR_WRITE; - p->processed += num; - p->buf = p->bufBase; + const size_t num = (size_t)(p->buf - p->bufBase); + if (p->res == SZ_OK) + { + if (num != ISeqOutStream_Write(p->outStream, p->bufBase, num)) + p->res = SZ_ERROR_WRITE; + } + p->processed += num; + p->buf = p->bufBase; } -static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) +MY_NO_INLINE static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) { - if ((UInt32)p->low < (UInt32)0xFF000000 || (int)(p->low >> 32) != 0) + UInt32 low = (UInt32)p->low; + unsigned high = (unsigned)(p->low >> 32); + p->low = (UInt32)(low << 8); + if (low < (UInt32)0xFF000000 || high != 0) + { { - Byte temp = p->cache; - do - { - Byte *buf = p->buf; - *buf++ = (Byte)(temp + (Byte)(p->low >> 32)); - p->buf = buf; - if (buf == p->bufLim) - RangeEnc_FlushStream(p); - temp = 0xFF; - } while (--p->cacheSize != 0); - p->cache = (Byte)((UInt32)p->low >> 24); + Byte *buf = p->buf; + *buf++ = (Byte)(p->cache + high); + p->cache = (unsigned)(low >> 24); + p->buf = buf; + if (buf == p->bufLim) + RangeEnc_FlushStream(p); + if (p->cacheSize == 0) + return; } - p->cacheSize++; - p->low = (UInt32)p->low << 8; + high += 0xFF; + for (;;) + { + Byte *buf = p->buf; + *buf++ = (Byte)(high); + p->buf = buf; + if (buf == p->bufLim) + RangeEnc_FlushStream(p); + if (--p->cacheSize == 0) + return; + } + } + p->cacheSize++; } static void RangeEnc_FlushData(CRangeEnc *p) { - int i; - for (i = 0; i < 5; i++) - RangeEnc_ShiftLow(p); + int i; + for (i = 0; i < 5; i++) + RangeEnc_ShiftLow(p); } -static void RangeEnc_EncodeDirectBits(CRangeEnc *p, UInt32 value, int numBits) +#define RC_NORM(p) if (range < kTopValue) { range <<= 8; RangeEnc_ShiftLow(p); } + +#define RC_BIT_PRE(p, prob) \ + ttt = *(prob); \ + newBound = (range >> kNumBitModelTotalBits) * ttt; + +// #define _LZMA_ENC_USE_BRANCH + +#ifdef _LZMA_ENC_USE_BRANCH + +#define RC_BIT(p, prob, bit) { \ + RC_BIT_PRE(p, prob) \ + if (bit == 0) { range = newBound; ttt += (kBitModelTotal - ttt) >> kNumMoveBits; } \ + else { (p)->low += newBound; range -= newBound; ttt -= ttt >> kNumMoveBits; } \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ + } + +#else + +#define RC_BIT(p, prob, bit) { \ + UInt32 mask; \ + RC_BIT_PRE(p, prob) \ + mask = 0 - (UInt32)bit; \ + range &= mask; \ + mask &= newBound; \ + range -= mask; \ + (p)->low += mask; \ + mask = (UInt32)bit - 1; \ + range += newBound & mask; \ + mask &= (kBitModelTotal - ((1 << kNumMoveBits) - 1)); \ + mask += ((1 << kNumMoveBits) - 1); \ + ttt += (UInt32)((Int32)(mask - ttt) >> kNumMoveBits); \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ + } + +#endif + + + + +#define RC_BIT_0_BASE(p, prob) \ + range = newBound; *(prob) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); + +#define RC_BIT_1_BASE(p, prob) \ + range -= newBound; (p)->low += newBound; *(prob) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); \ + +#define RC_BIT_0(p, prob) \ + RC_BIT_0_BASE(p, prob) \ + RC_NORM(p) + +#define RC_BIT_1(p, prob) \ + RC_BIT_1_BASE(p, prob) \ + RC_NORM(p) + +static void RangeEnc_EncodeBit_0(CRangeEnc *p, CLzmaProb *prob) { - do - { - p->range >>= 1; - p->low += p->range & (0 - ((value >> --numBits) & 1)); - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); - } - } while (numBits != 0); + UInt32 range, ttt, newBound; + range = p->range; + RC_BIT_PRE(p, prob) + RC_BIT_0(p, prob) + p->range = range; } -static void RangeEnc_EncodeBit(CRangeEnc *p, CLzmaProb *prob, UInt32 symbol) +static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 sym) { - UInt32 ttt = *prob; - UInt32 newBound = (p->range >> kNumBitModelTotalBits) * ttt; - if (symbol == 0) + UInt32 range = p->range; + sym |= 0x100; + do + { + UInt32 ttt, newBound; + // RangeEnc_EncodeBit(p, probs + (sym >> 8), (sym >> 7) & 1); + CLzmaProb *prob = probs + (sym >> 8); + UInt32 bit = (sym >> 7) & 1; + sym <<= 1; + RC_BIT(p, prob, bit); + } + while (sym < 0x10000); + p->range = range; +} + +static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 sym, UInt32 matchByte) +{ + UInt32 range = p->range; + UInt32 offs = 0x100; + sym |= 0x100; + do + { + UInt32 ttt, newBound; + CLzmaProb *prob; + UInt32 bit; + matchByte <<= 1; + // RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (sym >> 8)), (sym >> 7) & 1); + prob = probs + (offs + (matchByte & offs) + (sym >> 8)); + bit = (sym >> 7) & 1; + sym <<= 1; + offs &= ~(matchByte ^ sym); + RC_BIT(p, prob, bit); + } + while (sym < 0x10000); + p->range = range; +} + + + +static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) +{ + UInt32 i; + for (i = 0; i < (kBitModelTotal >> kNumMoveReducingBits); i++) + { + const unsigned kCyclesBits = kNumBitPriceShiftBits; + UInt32 w = (i << kNumMoveReducingBits) + (1 << (kNumMoveReducingBits - 1)); + unsigned bitCount = 0; + unsigned j; + for (j = 0; j < kCyclesBits; j++) { - p->range = newBound; - ttt += (kBitModelTotal - ttt) >> kNumMoveBits; - } - else - { - p->low += newBound; - p->range -= newBound; - ttt -= ttt >> kNumMoveBits; - } - *prob = (CLzmaProb)ttt; - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); + w = w * w; + bitCount <<= 1; + while (w >= ((UInt32)1 << 16)) + { + w >>= 1; + bitCount++; + } } + ProbPrices[i] = (CProbPrice)(((unsigned)kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + // printf("\n%3d: %5d", i, ProbPrices[i]); + } } -static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol) -{ - symbol |= 0x100; - do - { - RangeEnc_EncodeBit(p, probs + (symbol >> 8), (symbol >> 7) & 1); - symbol <<= 1; - } while (symbol < 0x10000); -} -static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol, UInt32 matchByte) -{ - UInt32 offs = 0x100; - symbol |= 0x100; - do - { - matchByte <<= 1; - RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (symbol >> 8)), (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); - } while (symbol < 0x10000); -} +#define GET_PRICE(prob, bit) \ + p->ProbPrices[((prob) ^ (unsigned)(((-(int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; -void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) -{ - UInt32 i; - for (i = (1 << kNumMoveReducingBits) / 2; i < kBitModelTotal; i += (1 << kNumMoveReducingBits)) - { - const int kCyclesBits = kNumBitPriceShiftBits; - UInt32 w = i; - UInt32 bitCount = 0; - int j; - for (j = 0; j < kCyclesBits; j++) - { - w = w * w; - bitCount <<= 1; - while (w >= ((UInt32)1 << 16)) - { - w >>= 1; - bitCount++; - } - } - ProbPrices[i >> kNumMoveReducingBits] = ((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); - } -} - -#define GET_PRICE(prob, symbol) \ - p->ProbPrices[((prob) ^ (((-(int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; - -#define GET_PRICEa(prob, symbol) \ - ProbPrices[((prob) ^ ((-((int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; +#define GET_PRICEa(prob, bit) \ + ProbPrices[((prob) ^ (unsigned)((-((int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; #define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] #define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -#define GET_PRICE_0a(prob) ProbPrices[(prob) >> kNumMoveReducingBits] -#define GET_PRICE_1a(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] +#define GET_PRICEa_0(prob) ProbPrices[(prob) >> kNumMoveReducingBits] +#define GET_PRICEa_1(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 symbol, UInt32 *ProbPrices) + +static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 sym, const CProbPrice *ProbPrices) { - UInt32 price = 0; - symbol |= 0x100; - do - { - price += GET_PRICEa(probs[symbol >> 8], (symbol >> 7) & 1); - symbol <<= 1; - } while (symbol < 0x10000); - return price; + UInt32 price = 0; + sym |= 0x100; + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); + } + while (sym >= 2); + return price; } -static UInt32 LitEnc_GetPriceMatched(const CLzmaProb *probs, UInt32 symbol, UInt32 matchByte, UInt32 *ProbPrices) + +static UInt32 LitEnc_Matched_GetPrice(const CLzmaProb *probs, UInt32 sym, UInt32 matchByte, const CProbPrice *ProbPrices) { - UInt32 price = 0; - UInt32 offs = 0x100; - symbol |= 0x100; - do - { - matchByte <<= 1; - price += GET_PRICEa(probs[offs + (matchByte & offs) + (symbol >> 8)], (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); - } while (symbol < 0x10000); - return price; + UInt32 price = 0; + UInt32 offs = 0x100; + sym |= 0x100; + do + { + matchByte <<= 1; + price += GET_PRICEa(probs[offs + (matchByte & offs) + (sym >> 8)], (sym >> 7) & 1); + sym <<= 1; + offs &= ~(matchByte ^ sym); + } + while (sym < 0x10000); + return price; } -static void RcTree_Encode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) + +static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, unsigned numBits, unsigned sym) { - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0;) - { - UInt32 bit; - i--; - bit = (symbol >> i) & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); - m = (m << 1) | bit; - } + UInt32 range = rc->range; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + unsigned bit = sym & 1; + // RangeEnc_EncodeBit(rc, probs + m, bit); + sym >>= 1; + RC_BIT(rc, probs + m, bit); + m = (m << 1) | bit; + } + while (--numBits); + rc->range = range; } -static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) -{ - UInt32 m = 1; - int i; - for (i = 0; i < numBitLevels; i++) - { - UInt32 bit = symbol & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); - m = (m << 1) | bit; - symbol >>= 1; - } -} -static UInt32 RcTree_GetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, UInt32 *ProbPrices) -{ - UInt32 price = 0; - symbol |= (1 << numBitLevels); - while (symbol != 1) - { - price += GET_PRICEa(probs[symbol >> 1], symbol & 1); - symbol >>= 1; - } - return price; -} - -static UInt32 RcTree_ReverseGetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, UInt32 *ProbPrices) -{ - UInt32 price = 0; - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0; i--) - { - UInt32 bit = symbol & 1; - symbol >>= 1; - price += GET_PRICEa(probs[m], bit); - m = (m << 1) | bit; - } - return price; -} static void LenEnc_Init(CLenEnc *p) { - unsigned i; - p->choice = p->choice2 = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumLowBits); i++) - p->low[i] = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumMidBits); i++) - p->mid[i] = kProbInitValue; - for (i = 0; i < kLenNumHighSymbols; i++) - p->high[i] = kProbInitValue; + unsigned i; + for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)); i++) + p->low[i] = kProbInitValue; + for (i = 0; i < kLenNumHighSymbols; i++) + p->high[i] = kProbInitValue; } -static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState) +static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned sym, unsigned posState) { - if (symbol < kLenNumLowSymbols) + UInt32 range, ttt, newBound; + CLzmaProb *probs = p->low; + range = rc->range; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols) + { + RC_BIT_1(rc, probs); + probs += kLenNumLowSymbols; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols * 2) { - RangeEnc_EncodeBit(rc, &p->choice, 0); - RcTree_Encode(rc, p->low + (posState << kLenNumLowBits), kLenNumLowBits, symbol); - } - else - { - RangeEnc_EncodeBit(rc, &p->choice, 1); - if (symbol < kLenNumLowSymbols + kLenNumMidSymbols) - { - RangeEnc_EncodeBit(rc, &p->choice2, 0); - RcTree_Encode(rc, p->mid + (posState << kLenNumMidBits), kLenNumMidBits, symbol - kLenNumLowSymbols); - } - else - { - RangeEnc_EncodeBit(rc, &p->choice2, 1); - RcTree_Encode(rc, p->high, kLenNumHighBits, symbol - kLenNumLowSymbols - kLenNumMidSymbols); - } + RC_BIT_1(rc, probs); + rc->range = range; + // RcTree_Encode(rc, p->high, kLenNumHighBits, sym - kLenNumLowSymbols * 2); + LitEnc_Encode(rc, p->high, sym - kLenNumLowSymbols * 2); + return; } + sym -= kLenNumLowSymbols; + } + + // RcTree_Encode(rc, probs + (posState << kLenNumLowBits), kLenNumLowBits, sym); + { + unsigned m; + unsigned bit; + RC_BIT_0(rc, probs); + probs += (posState << (1 + kLenNumLowBits)); + bit = (sym >> 2) ; RC_BIT(rc, probs + 1, bit); m = (1 << 1) + bit; + bit = (sym >> 1) & 1; RC_BIT(rc, probs + m, bit); m = (m << 1) + bit; + bit = sym & 1; RC_BIT(rc, probs + m, bit); + rc->range = range; + } } -static void LenEnc_SetPrices(CLenEnc *p, UInt32 posState, UInt32 numSymbols, UInt32 *prices, UInt32 *ProbPrices) +static void SetPrices_3(const CLzmaProb *probs, UInt32 startPrice, UInt32 *prices, const CProbPrice *ProbPrices) { - UInt32 a0 = GET_PRICE_0a(p->choice); - UInt32 a1 = GET_PRICE_1a(p->choice); - UInt32 b0 = a1 + GET_PRICE_0a(p->choice2); - UInt32 b1 = a1 + GET_PRICE_1a(p->choice2); - UInt32 i = 0; - for (i = 0; i < kLenNumLowSymbols; i++) - { - if (i >= numSymbols) - return; - prices[i] = a0 + RcTree_GetPrice(p->low + (posState << kLenNumLowBits), kLenNumLowBits, i, ProbPrices); - } - for (; i < kLenNumLowSymbols + kLenNumMidSymbols; i++) - { - if (i >= numSymbols) - return; - prices[i] = b0 + RcTree_GetPrice(p->mid + (posState << kLenNumMidBits), kLenNumMidBits, i - kLenNumLowSymbols, ProbPrices); - } - for (; i < numSymbols; i++) - prices[i] = b1 + RcTree_GetPrice(p->high, kLenNumHighBits, i - kLenNumLowSymbols - kLenNumMidSymbols, ProbPrices); + unsigned i; + for (i = 0; i < 8; i += 2) + { + UInt32 price = startPrice; + UInt32 prob; + price += GET_PRICEa(probs[1 ], (i >> 2)); + price += GET_PRICEa(probs[2 + (i >> 2)], (i >> 1) & 1); + prob = probs[4 + (i >> 1)]; + prices[i ] = price + GET_PRICEa_0(prob); + prices[i + 1] = price + GET_PRICEa_1(prob); + } } -static void MY_FAST_CALL LenPriceEnc_UpdateTable(CLenPriceEnc *p, UInt32 posState, UInt32 *ProbPrices) -{ - LenEnc_SetPrices(&p->p, posState, p->tableSize, p->prices[posState], ProbPrices); - p->counters[posState] = p->tableSize; -} -static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, UInt32 numPosStates, UInt32 *ProbPrices) +MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTables( + CLenPriceEnc *p, + unsigned numPosStates, + const CLenEnc *enc, + const CProbPrice *ProbPrices) { - UInt32 posState; + UInt32 b; + + { + unsigned prob = enc->low[0]; + UInt32 a, c; + unsigned posState; + b = GET_PRICEa_1(prob); + a = GET_PRICEa_0(prob); + c = b + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); for (posState = 0; posState < numPosStates; posState++) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} - -static void LenEnc_Encode2(CLenPriceEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState, Bool updatePrice, UInt32 *ProbPrices) -{ - LenEnc_Encode(&p->p, rc, symbol, posState); - if (updatePrice) - if (--p->counters[posState] == 0) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} - -static void MovePos(CLzmaEnc *p, UInt32 num) -{ -#ifdef SHOW_STAT - ttt += num; - printf("\n MovePos %d", num); -#endif - if (num != 0) { - p->additionalOffset += num; - p->matchFinder.Skip(p->matchFinderObj, num); + UInt32 *prices = p->prices[posState]; + const CLzmaProb *probs = enc->low + (posState << (1 + kLenNumLowBits)); + SetPrices_3(probs, a, prices, ProbPrices); + SetPrices_3(probs + kLenNumLowSymbols, c, prices + kLenNumLowSymbols, ProbPrices); } -} + } -static UInt32 ReadMatchDistances(CLzmaEnc *p, UInt32 *numDistancePairsRes) -{ - UInt32 lenRes = 0, numPairs; - p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); - numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); -#ifdef SHOW_STAT - printf("\n i = %d numPairs = %d ", ttt, numPairs / 2); - ttt++; + /* + { + unsigned i; + UInt32 b; + a = GET_PRICEa_0(enc->low[0]); + for (i = 0; i < kLenNumLowSymbols; i++) + p->prices2[i] = a; + a = GET_PRICEa_1(enc->low[0]); + b = a + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (i = kLenNumLowSymbols; i < kLenNumLowSymbols * 2; i++) + p->prices2[i] = b; + a += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + } + */ + + // p->counter = numSymbols; + // p->counter = 64; + + { + unsigned i = p->tableSize; + + if (i > kLenNumLowSymbols * 2) { - UInt32 i; - for (i = 0; i < numPairs; i += 2) - printf("%2d %6d | ", p->matches[i], p->matches[i + 1]); - } -#endif - if (numPairs > 0) - { - lenRes = p->matches[numPairs - 2]; - if (lenRes == p->numFastBytes) + const CLzmaProb *probs = enc->high; + UInt32 *prices = p->prices[0] + kLenNumLowSymbols * 2; + i -= kLenNumLowSymbols * 2 - 1; + i >>= 1; + b += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + do + { + /* + p->prices2[i] = a + + // RcTree_GetPrice(enc->high, kLenNumHighBits, i - kLenNumLowSymbols * 2, ProbPrices); + LitEnc_GetPrice(probs, i - kLenNumLowSymbols * 2, ProbPrices); + */ + // UInt32 price = a + RcTree_GetPrice(probs, kLenNumHighBits - 1, sym, ProbPrices); + unsigned sym = --i + (1 << (kLenNumHighBits - 1)); + UInt32 price = b; + do { - const Byte *pby = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - UInt32 distance = p->matches[numPairs - 1] + 1; - UInt32 numAvail = p->numAvail; - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - { - const Byte *pby2 = pby - distance; - for (; lenRes < numAvail && pby[lenRes] == pby2[lenRes]; lenRes++); - } + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); } + while (sym >= 2); + + { + unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; + prices[(size_t)i * 2 ] = price + GET_PRICEa_0(prob); + prices[(size_t)i * 2 + 1] = price + GET_PRICEa_1(prob); + } + } + while (i); + + { + unsigned posState; + size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); + for (posState = 1; posState < numPosStates; posState++) + memcpy(p->prices[posState] + kLenNumLowSymbols * 2, p->prices[0] + kLenNumLowSymbols * 2, num); + } } - p->additionalOffset++; - *numDistancePairsRes = numPairs; - return lenRes; + } } -#define MakeAsChar(p) (p)->backPrev = (UInt32)(-1); (p)->prev1IsChar = False; -#define MakeAsShortRep(p) (p)->backPrev = 0; (p)->prev1IsChar = False; -#define IsShortRep(p) ((p)->backPrev == 0) +/* + #ifdef SHOW_STAT + g_STAT_OFFSET += num; + printf("\n MovePos %u", num); + #endif +*/ + +#define MOVE_POS(p, num) { \ + p->additionalOffset += (num); \ + p->matchFinder.Skip(p->matchFinderObj, (UInt32)(num)); } -static UInt32 GetRepLen1Price(CLzmaEnc *p, UInt32 state, UInt32 posState) -{ - return - GET_PRICE_0(p->isRepG0[state]) + - GET_PRICE_0(p->isRep0Long[state][posState]); -} -static UInt32 GetPureRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 state, UInt32 posState) +static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) { - UInt32 price; - if (repIndex == 0) + unsigned numPairs; + + p->additionalOffset++; + p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); + { + const UInt32 *d = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); + // if (!d) { p->mf_Failure = True; *numPairsRes = 0; return 0; } + numPairs = (unsigned)(d - p->matches); + } + *numPairsRes = numPairs; + + #ifdef SHOW_STAT + printf("\n i = %u numPairs = %u ", g_STAT_OFFSET, numPairs / 2); + g_STAT_OFFSET++; + { + unsigned i; + for (i = 0; i < numPairs; i += 2) + printf("%2u %6u | ", p->matches[i], p->matches[i + 1]); + } + #endif + + if (numPairs == 0) + return 0; + { + const unsigned len = p->matches[(size_t)numPairs - 2]; + if (len != p->numFastBytes) + return len; { - price = GET_PRICE_0(p->isRepG0[state]); - price += GET_PRICE_1(p->isRep0Long[state][posState]); + UInt32 numAvail = p->numAvail; + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + { + const Byte *p1 = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + const Byte *p2 = p1 + len; + const ptrdiff_t dif = (ptrdiff_t)-1 - (ptrdiff_t)p->matches[(size_t)numPairs - 1]; + const Byte *lim = p1 + numAvail; + for (; p2 != lim && *p2 == p2[dif]; p2++) + {} + return (unsigned)(p2 - p1); + } } + } +} + +#define MARK_LIT ((UInt32)(Int32)-1) + +#define MakeAs_Lit(p) { (p)->dist = MARK_LIT; (p)->extra = 0; } +#define MakeAs_ShortRep(p) { (p)->dist = 0; (p)->extra = 0; } +#define IsShortRep(p) ((p)->dist == 0) + + +#define GetPrice_ShortRep(p, state, posState) \ + ( GET_PRICE_0(p->isRepG0[state]) + GET_PRICE_0(p->isRep0Long[state][posState])) + +#define GetPrice_Rep_0(p, state, posState) ( \ + GET_PRICE_1(p->isMatch[state][posState]) \ + + GET_PRICE_1(p->isRep0Long[state][posState])) \ + + GET_PRICE_1(p->isRep[state]) \ + + GET_PRICE_0(p->isRepG0[state]) + +MY_FORCE_INLINE +static UInt32 GetPrice_PureRep(const CLzmaEnc *p, unsigned repIndex, size_t state, size_t posState) +{ + UInt32 price; + UInt32 prob = p->isRepG0[state]; + if (repIndex == 0) + { + price = GET_PRICE_0(prob); + price += GET_PRICE_1(p->isRep0Long[state][posState]); + } + else + { + price = GET_PRICE_1(prob); + prob = p->isRepG1[state]; + if (repIndex == 1) + price += GET_PRICE_0(prob); else { - price = GET_PRICE_1(p->isRepG0[state]); - if (repIndex == 1) - price += GET_PRICE_0(p->isRepG1[state]); - else - { - price += GET_PRICE_1(p->isRepG1[state]); - price += GET_PRICE(p->isRepG2[state], repIndex - 2); - } + price += GET_PRICE_1(prob); + price += GET_PRICE(p->isRepG2[state], repIndex - 2); } - return price; + } + return price; } -static UInt32 GetRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 len, UInt32 state, UInt32 posState) + +static unsigned Backward(CLzmaEnc *p, unsigned cur) { - return p->repLenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN] + - GetPureRepPrice(p, repIndex, state, posState); -} + unsigned wr = cur + 1; + p->optEnd = wr; -static UInt32 Backward(CLzmaEnc *p, UInt32 *backRes, UInt32 cur) -{ - UInt32 posMem = p->opt[cur].posPrev; - UInt32 backMem = p->opt[cur].backPrev; - p->optimumEndIndex = cur; - do + for (;;) + { + UInt32 dist = p->opt[cur].dist; + unsigned len = (unsigned)p->opt[cur].len; + unsigned extra = (unsigned)p->opt[cur].extra; + cur -= len; + + if (extra) { - if (p->opt[cur].prev1IsChar) - { - MakeAsChar(&p->opt[posMem]) - p->opt[posMem].posPrev = posMem - 1; - if (p->opt[cur].prev2) - { - p->opt[posMem - 1].prev1IsChar = False; - p->opt[posMem - 1].posPrev = p->opt[cur].posPrev2; - p->opt[posMem - 1].backPrev = p->opt[cur].backPrev2; - } - } - { - UInt32 posPrev = posMem; - UInt32 backCur = backMem; - - backMem = p->opt[posPrev].backPrev; - posMem = p->opt[posPrev].posPrev; - - p->opt[posPrev].backPrev = backCur; - p->opt[posPrev].posPrev = cur; - cur = posPrev; + wr--; + p->opt[wr].len = (UInt32)len; + cur -= extra; + len = extra; + if (extra == 1) + { + p->opt[wr].dist = dist; + dist = MARK_LIT; + } + else + { + p->opt[wr].dist = 0; + len--; + wr--; + p->opt[wr].dist = MARK_LIT; + p->opt[wr].len = 1; + } } - } while (cur != 0); - *backRes = p->opt[0].backPrev; - p->optimumCurrentIndex = p->opt[0].posPrev; - return p->optimumCurrentIndex; + + if (cur == 0) + { + p->backRes = dist; + p->optCur = wr; + return len; + } + + wr--; + p->opt[wr].dist = dist; + p->opt[wr].len = (UInt32)len; + } } -#define LIT_PROBS(pos, prevByte) (p->litProbs + ((((pos) & p->lpMask) << p->lc) + ((prevByte) >> (8 - p->lc))) * 0x300) -static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) + +#define LIT_PROBS(pos, prevByte) \ + (p->litProbs + (UInt32)3 * (((((pos) << 8) + (prevByte)) & p->lpMask) << p->lc)) + + +static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) { - UInt32 numAvail, mainLen, numPairs, repMaxIndex, i, posState, lenEnd, len, cur; - UInt32 matchPrice, repMatchPrice, normalMatchPrice; - UInt32 reps[LZMA_NUM_REPS], repLens[LZMA_NUM_REPS]; - UInt32 *matches; + unsigned last, cur; + UInt32 reps[LZMA_NUM_REPS]; + unsigned repLens[LZMA_NUM_REPS]; + UInt32 *matches; + + { + UInt32 numAvail; + unsigned numPairs, mainLen, repMaxIndex, i, posState; + UInt32 matchPrice, repMatchPrice; const Byte *data; Byte curByte, matchByte; - if (p->optimumEndIndex != p->optimumCurrentIndex) - { - const COptimal *opt = &p->opt[p->optimumCurrentIndex]; - UInt32 lenRes = opt->posPrev - p->optimumCurrentIndex; - *backRes = opt->backPrev; - p->optimumCurrentIndex = opt->posPrev; - return lenRes; - } - p->optimumCurrentIndex = p->optimumEndIndex = 0; - + + p->optCur = p->optEnd = 0; + if (p->additionalOffset == 0) - mainLen = ReadMatchDistances(p, &numPairs); + mainLen = ReadMatchDistances(p, &numPairs); else { - mainLen = p->longestMatchLength; - numPairs = p->numPairs; + mainLen = p->longestMatchLen; + numPairs = p->numPairs; } - + numAvail = p->numAvail; if (numAvail < 2) { - *backRes = (UInt32)(-1); - return 1; + p->backRes = MARK_LIT; + return 1; } if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - + numAvail = LZMA_MATCH_LEN_MAX; + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; repMaxIndex = 0; + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 lenTest; - const Byte *data2; - reps[i] = p->reps[i]; - data2 = data - (reps[i] + 1); - if (data[0] != data2[0] || data[1] != data2[1]) - { - repLens[i] = 0; - continue; - } - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - repLens[i] = lenTest; - if (lenTest > repLens[repMaxIndex]) - repMaxIndex = i; + unsigned len; + const Byte *data2; + reps[i] = p->reps[i]; + data2 = data - reps[i]; + if (data[0] != data2[0] || data[1] != data2[1]) + { + repLens[i] = 0; + continue; + } + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + repLens[i] = len; + if (len > repLens[repMaxIndex]) + repMaxIndex = i; + if (len == LZMA_MATCH_LEN_MAX) // 21.03 : optimization + break; } + if (repLens[repMaxIndex] >= p->numFastBytes) { - UInt32 lenRes; - *backRes = repMaxIndex; - lenRes = repLens[repMaxIndex]; - MovePos(p, lenRes - 1); - return lenRes; + unsigned len; + p->backRes = (UInt32)repMaxIndex; + len = repLens[repMaxIndex]; + MOVE_POS(p, len - 1) + return len; } - + matches = p->matches; + #define MATCHES matches + // #define MATCHES p->matches + if (mainLen >= p->numFastBytes) { - *backRes = matches[numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); - return mainLen; + p->backRes = MATCHES[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) + return mainLen; } + curByte = *data; - matchByte = *(data - (reps[0] + 1)); + matchByte = *(data - reps[0]); - if (mainLen < 2 && curByte != matchByte && repLens[repMaxIndex] < 2) + last = repLens[repMaxIndex]; + if (last <= mainLen) + last = mainLen; + + if (last < 2 && curByte != matchByte) { - *backRes = (UInt32)-1; - return 1; + p->backRes = MARK_LIT; + return 1; } - + p->opt[0].state = (CState)p->state; - + posState = (position & p->pbMask); - + { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + - (!IsCharState(p->state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + + (!IsLitState(p->state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); } - MakeAsChar(&p->opt[1]); - + MakeAs_Lit(&p->opt[1]); + matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); - - if (matchByte == curByte) + + // 18.06 + if (matchByte == curByte && repLens[0] == 0) { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, p->state, posState); - if (shortRepPrice < p->opt[1].price) - { - p->opt[1].price = shortRepPrice; - MakeAsShortRep(&p->opt[1]); - } - } - lenEnd = ((mainLen >= repLens[repMaxIndex]) ? mainLen : repLens[repMaxIndex]); - - if (lenEnd < 2) - { - *backRes = p->opt[1].backPrev; + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, p->state, posState); + if (shortRepPrice < p->opt[1].price) + { + p->opt[1].price = shortRepPrice; + MakeAs_ShortRep(&p->opt[1]); + } + if (last < 2) + { + p->backRes = p->opt[1].dist; return 1; + } } - - p->opt[1].posPrev = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - p->opt[0].backs[i] = reps[i]; - - len = lenEnd; - do - p->opt[len--].price = kInfinityPrice; - while (len >= 2); - + + p->opt[1].len = 1; + + p->opt[0].reps[0] = reps[0]; + p->opt[0].reps[1] = reps[1]; + p->opt[0].reps[2] = reps[2]; + p->opt[0].reps[3] = reps[3]; + + // ---------- REP ---------- + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 repLen = repLens[i]; - UInt32 price; - if (repLen < 2) - continue; - price = repMatchPrice + GetPureRepPrice(p, i, p->state, posState); - do + unsigned repLen = repLens[i]; + UInt32 price; + if (repLen < 2) + continue; + price = repMatchPrice + GetPrice_PureRep(p, i, p->state, posState); + do + { + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, repLen); + COptimal *opt = &p->opt[repLen]; + if (price2 < opt->price) { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][repLen - 2]; - COptimal *opt = &p->opt[repLen]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = i; - opt->prev1IsChar = False; - } - } while (--repLen >= 2); - } - - normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); - - len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); - if (len <= mainLen) - { - UInt32 offs = 0; - while (len > matches[offs]) - offs += 2; - for (;; len++) - { - COptimal *opt; - UInt32 distance = matches[offs + 1]; - - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN]; - UInt32 lenToPosState = GetLenToPosState(len); - if (distance < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][distance]; - else - { - UInt32 slot; - GetPosSlot2(distance, slot); - curAndLenPrice += p->alignPrices[distance & kAlignMask] + p->posSlotPrices[lenToPosState][slot]; - } - opt = &p->opt[len]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = distance + LZMA_NUM_REPS; - opt->prev1IsChar = False; - } - if (len == matches[offs]) - { - offs += 2; - if (offs == numPairs) - break; - } + opt->price = price2; + opt->len = (UInt32)repLen; + opt->dist = (UInt32)i; + opt->extra = 0; } + } + while (--repLen >= 2); } + + + // ---------- MATCH ---------- + { + unsigned len = repLens[0] + 1; + if (len <= mainLen) + { + unsigned offs = 0; + UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); + + if (len < 2) + len = 2; + else + while (len > MATCHES[offs]) + offs += 2; + + for (; ; len++) + { + COptimal *opt; + UInt32 dist = MATCHES[(size_t)offs + 1]; + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); + unsigned lenToPosState = GetLenToPosState(len); + + if (dist < kNumFullDistances) + price += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; + else + { + unsigned slot; + GetPosSlot2(dist, slot); + price += p->alignPrices[dist & kAlignMask]; + price += p->posSlotPrices[lenToPosState][slot]; + } + + opt = &p->opt[len]; + + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } + + if (len == MATCHES[offs]) + { + offs += 2; + if (offs == numPairs) + break; + } + } + } + } + cur = 0; -#ifdef SHOW_STAT2 - if (position >= 0) + #ifdef SHOW_STAT2 + /* if (position >= 0) */ { - unsigned i; - printf("\n pos = %4X", position); - for (i = cur; i <= lenEnd; i++) - printf("\nprice[%4X] = %d", position - cur + i, p->opt[i].price); + unsigned i; + printf("\n pos = %4X", position); + for (i = cur; i <= last; i++) + printf("\nprice[%4X] = %u", position - cur + i, p->opt[i].price); } -#endif + #endif + } - for (;;) + + + // ---------- Optimal Parsing ---------- + + for (;;) + { + unsigned numAvail; + UInt32 numAvailFull; + unsigned newLen, numPairs, prev, state, posState, startLen; + UInt32 litPrice, matchPrice, repMatchPrice; + BoolInt nextIsLit; + Byte curByte, matchByte; + const Byte *data; + COptimal *curOpt, *nextOpt; + + if (++cur == last) + break; + + // 18.06 + if (cur >= kNumOpts - 64) { - UInt32 numAvailFull, newLen, numPairs, posPrev, state, posState, startLen; - UInt32 curPrice, curAnd1Price, matchPrice, repMatchPrice; - Bool nextIsChar; - Byte curByte, matchByte; - const Byte *data; - COptimal *curOpt; - COptimal *nextOpt; - - cur++; - if (cur == lenEnd) - return Backward(p, backRes, cur); - - newLen = ReadMatchDistances(p, &numPairs); - if (newLen >= p->numFastBytes) + unsigned j, best; + UInt32 price = p->opt[cur].price; + best = cur; + for (j = cur + 1; j <= last; j++) + { + UInt32 price2 = p->opt[j].price; + if (price >= price2) { - p->numPairs = numPairs; - p->longestMatchLength = newLen; - return Backward(p, backRes, cur); + price = price2; + best = j; } - position++; - curOpt = &p->opt[cur]; - posPrev = curOpt->posPrev; - if (curOpt->prev1IsChar) + } + { + unsigned delta = best - cur; + if (delta != 0) { - posPrev--; - if (curOpt->prev2) - { - state = p->opt[curOpt->posPrev2].state; - if (curOpt->backPrev2 < LZMA_NUM_REPS) - state = kRepNextStates[state]; - else - state = kMatchNextStates[state]; - } - else - state = p->opt[posPrev].state; - state = kLiteralNextStates[state]; + MOVE_POS(p, delta); } + } + cur = best; + break; + } + + newLen = ReadMatchDistances(p, &numPairs); + + if (newLen >= p->numFastBytes) + { + p->numPairs = numPairs; + p->longestMatchLen = newLen; + break; + } + + curOpt = &p->opt[cur]; + + position++; + + // we need that check here, if skip_items in p->opt are possible + /* + if (curOpt->price >= kInfinityPrice) + continue; + */ + + prev = cur - curOpt->len; + + if (curOpt->len == 1) + { + state = (unsigned)p->opt[prev].state; + if (IsShortRep(curOpt)) + state = kShortRepNextStates[state]; + else + state = kLiteralNextStates[state]; + } + else + { + const COptimal *prevOpt; + UInt32 b0; + UInt32 dist = curOpt->dist; + + if (curOpt->extra) + { + prev -= (unsigned)curOpt->extra; + state = kState_RepAfterLit; + if (curOpt->extra == 1) + state = (dist < LZMA_NUM_REPS ? kState_RepAfterLit : kState_MatchAfterLit); + } + else + { + state = (unsigned)p->opt[prev].state; + if (dist < LZMA_NUM_REPS) + state = kRepNextStates[state]; else - state = p->opt[posPrev].state; - if (posPrev == cur - 1) + state = kMatchNextStates[state]; + } + + prevOpt = &p->opt[prev]; + b0 = prevOpt->reps[0]; + + if (dist < LZMA_NUM_REPS) + { + if (dist == 0) { - if (IsShortRep(curOpt)) - state = kShortRepNextStates[state]; - else - state = kLiteralNextStates[state]; + reps[0] = b0; + reps[1] = prevOpt->reps[1]; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; } else { - UInt32 pos; - const COptimal *prevOpt; - if (curOpt->prev1IsChar && curOpt->prev2) + reps[1] = b0; + b0 = prevOpt->reps[1]; + if (dist == 1) + { + reps[0] = b0; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[2] = b0; + reps[0] = prevOpt->reps[dist]; + reps[3] = prevOpt->reps[dist ^ 1]; + } + } + } + else + { + reps[0] = (dist - LZMA_NUM_REPS + 1); + reps[1] = b0; + reps[2] = prevOpt->reps[1]; + reps[3] = prevOpt->reps[2]; + } + } + + curOpt->state = (CState)state; + curOpt->reps[0] = reps[0]; + curOpt->reps[1] = reps[1]; + curOpt->reps[2] = reps[2]; + curOpt->reps[3] = reps[3]; + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + curByte = *data; + matchByte = *(data - reps[0]); + + posState = (position & p->pbMask); + + /* + The order of Price checks: + < LIT + <= SHORT_REP + < LIT : REP_0 + < REP [ : LIT : REP_0 ] + < MATCH [ : LIT : REP_0 ] + */ + + { + UInt32 curPrice = curOpt->price; + unsigned prob = p->isMatch[state][posState]; + matchPrice = curPrice + GET_PRICE_1(prob); + litPrice = curPrice + GET_PRICE_0(prob); + } + + nextOpt = &p->opt[(size_t)cur + 1]; + nextIsLit = False; + + // here we can allow skip_items in p->opt, if we don't check (nextOpt->price < kInfinityPrice) + // 18.new.06 + if ((nextOpt->price < kInfinityPrice + // && !IsLitState(state) + && matchByte == curByte) + || litPrice > nextOpt->price + ) + litPrice = 0; + else + { + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + litPrice += (!IsLitState(state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + + if (litPrice < nextOpt->price) + { + nextOpt->price = litPrice; + nextOpt->len = 1; + MakeAs_Lit(nextOpt); + nextIsLit = True; + } + } + + repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); + + numAvailFull = p->numAvail; + { + unsigned temp = kNumOpts - 1 - cur; + if (numAvailFull > temp) + numAvailFull = (UInt32)temp; + } + + // 18.06 + // ---------- SHORT_REP ---------- + if (IsLitState(state)) // 18.new + if (matchByte == curByte) + if (repMatchPrice < nextOpt->price) // 18.new + // if (numAvailFull < 2 || data[1] != *(data - reps[0] + 1)) + if ( + // nextOpt->price >= kInfinityPrice || + nextOpt->len < 2 // we can check nextOpt->len, if skip items are not allowed in p->opt + || (nextOpt->dist != 0 + // && nextOpt->extra <= 1 // 17.old + ) + ) + { + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, state, posState); + // if (shortRepPrice <= nextOpt->price) // 17.old + if (shortRepPrice < nextOpt->price) // 18.new + { + nextOpt->price = shortRepPrice; + nextOpt->len = 1; + MakeAs_ShortRep(nextOpt); + nextIsLit = False; + } + } + + if (numAvailFull < 2) + continue; + numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); + + // numAvail <= p->numFastBytes + + // ---------- LIT : REP_0 ---------- + + if (!nextIsLit + && litPrice != 0 // 18.new + && matchByte != curByte + && numAvailFull > 2) + { + const Byte *data2 = data - reps[0]; + if (data[1] == data2[1] && data[2] == data2[2]) + { + unsigned len; + unsigned limit = p->numFastBytes + 1; + if (limit > numAvailFull) + limit = numAvailFull; + for (len = 3; len < limit && data[len] == data2[len]; len++) + {} + + { + unsigned state2 = kLiteralNextStates[state]; + unsigned posState2 = (position + 1) & p->pbMask; + UInt32 price = litPrice + GetPrice_Rep_0(p, state2, posState2); + { + unsigned offset = cur + len; + + if (last < offset) + last = offset; + + // do { - posPrev = curOpt->posPrev2; - pos = curOpt->backPrev2; - state = kRepNextStates[state]; + UInt32 price2; + COptimal *opt; + len--; + // price2 = price + GetPrice_Len_Rep_0(p, len, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len); + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len; + opt->dist = 0; + opt->extra = 1; + } } - else + // while (len >= 3); + } + } + } + } + + startLen = 2; /* speed optimization */ + + { + // ---------- REP ---------- + unsigned repIndex = 0; // 17.old + // unsigned repIndex = IsLitState(state) ? 0 : 1; // 18.notused + for (; repIndex < LZMA_NUM_REPS; repIndex++) + { + unsigned len; + UInt32 price; + const Byte *data2 = data - reps[repIndex]; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + + // if (len < startLen) continue; // 18.new: speed optimization + + { + unsigned offset = cur + len; + if (last < offset) + last = offset; + } + { + unsigned len2 = len; + price = repMatchPrice + GetPrice_PureRep(p, repIndex, state, posState); + do + { + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, len2); + COptimal *opt = &p->opt[cur + len2]; + if (price2 < opt->price) { - pos = curOpt->backPrev; - if (pos < LZMA_NUM_REPS) - state = kRepNextStates[state]; - else - state = kMatchNextStates[state]; + opt->price = price2; + opt->len = (UInt32)len2; + opt->dist = (UInt32)repIndex; + opt->extra = 0; } - prevOpt = &p->opt[posPrev]; - if (pos < LZMA_NUM_REPS) + } + while (--len2 >= 2); + } + + if (repIndex == 0) startLen = len + 1; // 17.old + // startLen = len + 1; // 18.new + + /* if (_maxMode) */ + { + // ---------- REP : LIT : REP_0 ---------- + // numFastBytes + 1 + numFastBytes + + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; + if (limit > numAvailFull) + limit = numAvailFull; + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) + { + unsigned state2 = kRepNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + price += GET_PRICE_LEN(&p->repLenEnc, posState, len) + + GET_PRICE_0(p->isMatch[state2][posState2]) + + LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); + + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterRep; + posState2 = (posState2 + 1) & p->pbMask; + + + price += GetPrice_Rep_0(p, state2, posState2); + + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + // if (len2 >= 3) + { { - UInt32 i; - reps[0] = prevOpt->backs[pos]; - for (i = 1; i <= pos; i++) - reps[i] = prevOpt->backs[i - 1]; - for (; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i]; - } - else - { - UInt32 i; - reps[0] = (pos - LZMA_NUM_REPS); - for (i = 1; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i - 1]; - } - } - curOpt->state = (CState)state; + unsigned offset = cur + len + len2; - curOpt->backs[0] = reps[0]; - curOpt->backs[1] = reps[1]; - curOpt->backs[2] = reps[2]; - curOpt->backs[3] = reps[3]; - - curPrice = curOpt->price; - nextIsChar = False; - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - curByte = *data; - matchByte = *(data - (reps[0] + 1)); - - posState = (position & p->pbMask); - - curAnd1Price = curPrice + GET_PRICE_0(p->isMatch[state][posState]); - { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - curAnd1Price += - (!IsCharState(state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - } - - nextOpt = &p->opt[cur + 1]; - - if (curAnd1Price < nextOpt->price) - { - nextOpt->price = curAnd1Price; - nextOpt->posPrev = cur; - MakeAsChar(nextOpt); - nextIsChar = True; - } - - matchPrice = curPrice + GET_PRICE_1(p->isMatch[state][posState]); - repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); - - if (matchByte == curByte && !(nextOpt->posPrev < cur && nextOpt->backPrev == 0)) - { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, state, posState); - if (shortRepPrice <= nextOpt->price) - { - nextOpt->price = shortRepPrice; - nextOpt->posPrev = cur; - MakeAsShortRep(nextOpt); - nextIsChar = True; - } - } - numAvailFull = p->numAvail; - { - UInt32 temp = kNumOpts - 1 - cur; - if (temp < numAvailFull) - numAvailFull = temp; - } - - if (numAvailFull < 2) - continue; - numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); - - if (!nextIsChar && matchByte != curByte) /* speed optimization */ - { - /* try Literal + rep0 */ - UInt32 temp; - UInt32 lenTest2; - const Byte *data2 = data - (reps[0] + 1); - UInt32 limit = p->numFastBytes + 1; - if (limit > numAvailFull) - limit = numAvailFull; - - for (temp = 1; temp < limit && data[temp] == data2[temp]; temp++); - lenTest2 = temp - 1; - if (lenTest2 >= 2) - { - UInt32 state2 = kLiteralNextStates[state]; - UInt32 posStateNext = (position + 1) & p->pbMask; - UInt32 nextRepMatchPrice = curAnd1Price + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - /* for (; lenTest2 >= 2; lenTest2--) */ - { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = False; - } - } - } - } - - startLen = 2; /* speed optimization */ - { - UInt32 repIndex; - for (repIndex = 0; repIndex < LZMA_NUM_REPS; repIndex++) - { - UInt32 lenTest; - UInt32 lenTestTemp; - UInt32 price; - const Byte *data2 = data - (reps[repIndex] + 1); - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - while (lenEnd < cur + lenTest) - p->opt[++lenEnd].price = kInfinityPrice; - lenTestTemp = lenTest; - price = repMatchPrice + GetPureRepPrice(p, repIndex, state, posState); - do - { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][lenTest - 2]; - COptimal *opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = repIndex; - opt->prev1IsChar = False; - } - } while (--lenTest >= 2); - lenTest = lenTestTemp; - - if (repIndex == 0) - startLen = lenTest + 1; - - /* if (_maxMode) */ - { - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; - UInt32 nextRepMatchPrice; - if (limit > numAvailFull) - limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) - { - UInt32 state2 = kRepNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = - price + p->repLenEnc.prices[posState][lenTest - 2] + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (position + lenTest + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - - /* for (; lenTest2 >= 2; lenTest2--) */ - { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + lenTest + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = repIndex; - } - } - } - } - } - } - /* for (UInt32 lenTest = 2; lenTest <= newLen; lenTest++) */ - if (newLen > numAvail) - { - newLen = numAvail; - for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2); - matches[numPairs] = newLen; - numPairs += 2; - } - if (newLen >= startLen) - { - UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); - UInt32 offs, curBack, posSlot; - UInt32 lenTest; - while (lenEnd < cur + newLen) - p->opt[++lenEnd].price = kInfinityPrice; - - offs = 0; - while (startLen > matches[offs]) - offs += 2; - curBack = matches[offs + 1]; - GetPosSlot2(curBack, posSlot); - for (lenTest = /*2*/ startLen;; lenTest++) - { - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][lenTest - LZMA_MATCH_LEN_MIN]; - UInt32 lenToPosState = GetLenToPosState(lenTest); + if (last < offset) + last = offset; + // do + { + UInt32 price2; COptimal *opt; - if (curBack < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][curBack]; - else - curAndLenPrice += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[curBack & kAlignMask]; + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); - opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = curBack + LZMA_NUM_REPS; - opt->prev1IsChar = False; - } - - if (/*_maxMode && */lenTest == matches[offs]) - { - /* Try Match + Literal + Rep0 */ - const Byte *data2 = data - (curBack + 1); - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; - UInt32 nextRepMatchPrice; - if (limit > numAvailFull) - limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) - { - UInt32 state2 = kMatchNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = curAndLenPrice + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (posStateNext + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - - /* for (; lenTest2 >= 2; lenTest2--) */ - { - UInt32 offset = cur + lenTest + 1 + lenTest2; - UInt32 curAndLenPrice; - COptimal *opt; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = curBack + LZMA_NUM_REPS; - } - } - } - offs += 2; - if (offs == numPairs) - break; - curBack = matches[offs + 1]; - if (curBack >= kNumFullDistances) - GetPosSlot2(curBack, posSlot); + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = (UInt32)repIndex; } + } + // while (len2 >= 3); } + } + } } + } } + + + // ---------- MATCH ---------- + /* for (unsigned len = 2; len <= newLen; len++) */ + if (newLen > numAvail) + { + newLen = numAvail; + for (numPairs = 0; newLen > MATCHES[numPairs]; numPairs += 2); + MATCHES[numPairs] = (UInt32)newLen; + numPairs += 2; + } + + // startLen = 2; /* speed optimization */ + + if (newLen >= startLen) + { + UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); + UInt32 dist; + unsigned offs, posSlot, len; + + { + unsigned offset = cur + newLen; + if (last < offset) + last = offset; + } + + offs = 0; + while (startLen > MATCHES[offs]) + offs += 2; + dist = MATCHES[(size_t)offs + 1]; + + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); + + for (len = /*2*/ startLen; ; len++) + { + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); + { + COptimal *opt; + unsigned lenNorm = len - 2; + lenNorm = GetLenToPosState2(lenNorm); + if (dist < kNumFullDistances) + price += p->distancesPrices[lenNorm][dist & (kNumFullDistances - 1)]; + else + price += p->posSlotPrices[lenNorm][posSlot] + p->alignPrices[dist & kAlignMask]; + + opt = &p->opt[cur + len]; + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } + } + + if (len == MATCHES[offs]) + { + // if (p->_maxMode) { + // MATCH : LIT : REP_0 + + const Byte *data2 = data - dist - 1; + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; + if (limit > numAvailFull) + limit = numAvailFull; + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) + { + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + + // if (len2 >= 3) + { + unsigned state2 = kMatchNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + unsigned offset; + price += GET_PRICE_0(p->isMatch[state2][posState2]); + price += LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); + + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterMatch; + + posState2 = (posState2 + 1) & p->pbMask; + price += GetPrice_Rep_0(p, state2, posState2); + + offset = cur + len + len2; + + if (last < offset) + last = offset; + // do + { + UInt32 price2; + COptimal *opt; + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = dist + LZMA_NUM_REPS; + } + } + // while (len2 >= 3); + } + + } + + offs += 2; + if (offs == numPairs) + break; + dist = MATCHES[(size_t)offs + 1]; + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); + } + } + } + } + + do + p->opt[last].price = kInfinityPrice; + while (--last); + + return Backward(p, cur); } + + #define ChangePair(smallDist, bigDist) (((bigDist) >> 7) > (smallDist)) -static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) + + +static unsigned GetOptimumFast(CLzmaEnc *p) { - UInt32 numAvail, mainLen, mainDist, numPairs, repIndex, repLen, i; - const Byte *data; - const UInt32 *matches; + UInt32 numAvail, mainDist; + unsigned mainLen, numPairs, repIndex, repLen, i; + const Byte *data; - if (p->additionalOffset == 0) - mainLen = ReadMatchDistances(p, &numPairs); - else + if (p->additionalOffset == 0) + mainLen = ReadMatchDistances(p, &numPairs); + else + { + mainLen = p->longestMatchLen; + numPairs = p->numPairs; + } + + numAvail = p->numAvail; + p->backRes = MARK_LIT; + if (numAvail < 2) + return 1; + // if (mainLen < 2 && p->state == 0) return 1; // 18.06.notused + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + repLen = repIndex = 0; + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned len; + const Byte *data2 = data - p->reps[i]; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + if (len >= p->numFastBytes) { - mainLen = p->longestMatchLength; - numPairs = p->numPairs; + p->backRes = (UInt32)i; + MOVE_POS(p, len - 1) + return len; } - - numAvail = p->numAvail; - *backRes = (UInt32)-1; - if (numAvail < 2) - return 1; - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - - repLen = repIndex = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) + if (len > repLen) { - UInt32 len; - const Byte *data2 = data - (p->reps[i] + 1); - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - for (len = 2; len < numAvail && data[len] == data2[len]; len++); - if (len >= p->numFastBytes) - { - *backRes = i; - MovePos(p, len - 1); - return len; - } - if (len > repLen) - { - repIndex = i; - repLen = len; - } + repIndex = i; + repLen = len; } + } - matches = p->matches; - if (mainLen >= p->numFastBytes) - { - *backRes = matches[numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); - return mainLen; - } - - mainDist = 0; /* for GCC */ - if (mainLen >= 2) - { - mainDist = matches[numPairs - 1]; - while (numPairs > 2 && mainLen == matches[numPairs - 4] + 1) - { - if (!ChangePair(matches[numPairs - 3], mainDist)) - break; - numPairs -= 2; - mainLen = matches[numPairs - 2]; - mainDist = matches[numPairs - 1]; - } - if (mainLen == 2 && mainDist >= 0x80) - mainLen = 1; - } - - if (repLen >= 2 && ( - (repLen + 1 >= mainLen) || - (repLen + 2 >= mainLen && mainDist >= (1 << 9)) || - (repLen + 3 >= mainLen && mainDist >= (1 << 15)))) - { - *backRes = repIndex; - MovePos(p, repLen - 1); - return repLen; - } - - if (mainLen < 2 || numAvail <= 2) - return 1; - - p->longestMatchLength = ReadMatchDistances(p, &p->numPairs); - if (p->longestMatchLength >= 2) - { - UInt32 newDistance = matches[p->numPairs - 1]; - if ((p->longestMatchLength >= mainLen && newDistance < mainDist) || - (p->longestMatchLength == mainLen + 1 && !ChangePair(mainDist, newDistance)) || - (p->longestMatchLength > mainLen + 1) || - (p->longestMatchLength + 1 >= mainLen && mainLen >= 3 && ChangePair(newDistance, mainDist))) - return 1; - } - - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 len, limit; - const Byte *data2 = data - (p->reps[i] + 1); - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - limit = mainLen - 1; - for (len = 2; len < limit && data[len] == data2[len]; len++); - if (len >= limit) - return 1; - } - *backRes = mainDist + LZMA_NUM_REPS; - MovePos(p, mainLen - 2); + if (mainLen >= p->numFastBytes) + { + p->backRes = p->matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) return mainLen; + } + + mainDist = 0; /* for GCC */ + + if (mainLen >= 2) + { + mainDist = p->matches[(size_t)numPairs - 1]; + while (numPairs > 2) + { + UInt32 dist2; + if (mainLen != p->matches[(size_t)numPairs - 4] + 1) + break; + dist2 = p->matches[(size_t)numPairs - 3]; + if (!ChangePair(dist2, mainDist)) + break; + numPairs -= 2; + mainLen--; + mainDist = dist2; + } + if (mainLen == 2 && mainDist >= 0x80) + mainLen = 1; + } + + if (repLen >= 2) + if ( repLen + 1 >= mainLen + || (repLen + 2 >= mainLen && mainDist >= (1 << 9)) + || (repLen + 3 >= mainLen && mainDist >= (1 << 15))) + { + p->backRes = (UInt32)repIndex; + MOVE_POS(p, repLen - 1) + return repLen; + } + + if (mainLen < 2 || numAvail <= 2) + return 1; + + { + unsigned len1 = ReadMatchDistances(p, &p->numPairs); + p->longestMatchLen = len1; + + if (len1 >= 2) + { + UInt32 newDist = p->matches[(size_t)p->numPairs - 1]; + if ( (len1 >= mainLen && newDist < mainDist) + || (len1 == mainLen + 1 && !ChangePair(mainDist, newDist)) + || (len1 > mainLen + 1) + || (len1 + 1 >= mainLen && mainLen >= 3 && ChangePair(newDist, mainDist))) + return 1; + } + } + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned len, limit; + const Byte *data2 = data - p->reps[i]; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + limit = mainLen - 1; + for (len = 2;; len++) + { + if (len >= limit) + return 1; + if (data[len] != data2[len]) + break; + } + } + + p->backRes = mainDist + LZMA_NUM_REPS; + if (mainLen != 2) + { + MOVE_POS(p, mainLen - 2) + } + return mainLen; } -static void WriteEndMarker(CLzmaEnc *p, UInt32 posState) + + + +static void WriteEndMarker(CLzmaEnc *p, unsigned posState) { - UInt32 len; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); - p->state = kMatchNextStates[p->state]; - len = LZMA_MATCH_LEN_MIN; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, (1 << kNumPosSlotBits) - 1); - RangeEnc_EncodeDirectBits(&p->rc, (((UInt32)1 << 30) - 1) >> kNumAlignBits, 30 - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + UInt32 range; + range = p->rc.range; + { + UInt32 ttt, newBound; + CLzmaProb *prob = &p->isMatch[p->state][posState]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_1(&p->rc, prob) + prob = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_0(&p->rc, prob) + } + p->state = kMatchNextStates[p->state]; + + p->rc.range = range; + LenEnc_Encode(&p->lenProbs, &p->rc, 0, posState); + range = p->rc.range; + + { + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[0], (1 << kNumPosSlotBits) - 1); + CLzmaProb *probs = p->posSlotEncoder[0]; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < (1 << kNumPosSlotBits)); + } + { + // RangeEnc_EncodeDirectBits(&p->rc, ((UInt32)1 << (30 - kNumAlignBits)) - 1, 30 - kNumAlignBits); UInt32 range = p->range; + unsigned numBits = 30 - kNumAlignBits; + do + { + range >>= 1; + p->rc.low += range; + RC_NORM(&p->rc) + } + while (--numBits); + } + + { + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + CLzmaProb *probs = p->posAlignEncoder; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < kAlignTableSize); + } + p->rc.range = range; } + static SRes CheckErrors(CLzmaEnc *p) { - if (p->result != SZ_OK) - return p->result; - if (p->rc.res != SZ_OK) - p->result = SZ_ERROR_WRITE; - if (p->matchFinderBase.result != SZ_OK) - p->result = SZ_ERROR_READ; - if (p->result != SZ_OK) - p->finished = True; + if (p->result != SZ_OK) return p->result; -} + if (p->rc.res != SZ_OK) + p->result = SZ_ERROR_WRITE; -static SRes Flush(CLzmaEnc *p, UInt32 nowPos) -{ - /* ReleaseMFStream(); */ + #ifndef _7ZIP_ST + if ( + // p->mf_Failure || + (p->mtMode && + ( // p->matchFinderMt.failure_LZ_LZ || + p->matchFinderMt.failure_LZ_BT)) + ) + { + p->result = MY_HRES_ERROR__INTERNAL_ERROR; + // printf("\nCheckErrors p->matchFinderMt.failureLZ\n"); + } + #endif + + if (MFB.result != SZ_OK) + p->result = SZ_ERROR_READ; + + if (p->result != SZ_OK) p->finished = True; - if (p->writeEndMark) - WriteEndMarker(p, nowPos & p->pbMask); - RangeEnc_FlushData(&p->rc); - RangeEnc_FlushStream(&p->rc); - return CheckErrors(p); + return p->result; } -static void FillAlignPrices(CLzmaEnc *p) + +MY_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) { - UInt32 i; - for (i = 0; i < kAlignTableSize; i++) - p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); - p->alignPriceCount = 0; + /* ReleaseMFStream(); */ + p->finished = True; + if (p->writeEndMark) + WriteEndMarker(p, nowPos & p->pbMask); + RangeEnc_FlushData(&p->rc); + RangeEnc_FlushStream(&p->rc); + return CheckErrors(p); } -static void FillDistancesPrices(CLzmaEnc *p) + +MY_NO_INLINE static void FillAlignPrices(CLzmaEnc *p) { - UInt32 tempPrices[kNumFullDistances]; - UInt32 i, lenToPosState; - for (i = kStartPosModelIndex; i < kNumFullDistances; i++) + unsigned i; + const CProbPrice *ProbPrices = p->ProbPrices; + const CLzmaProb *probs = p->posAlignEncoder; + // p->alignPriceCount = 0; + for (i = 0; i < kAlignTableSize / 2; i++) + { + UInt32 price = 0; + unsigned sym = i; + unsigned m = 1; + unsigned bit; + UInt32 prob; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + prob = probs[m]; + p->alignPrices[i ] = price + GET_PRICEa_0(prob); + p->alignPrices[i + 8] = price + GET_PRICEa_1(prob); + // p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); + } +} + + +MY_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) +{ + // int y; for (y = 0; y < 100; y++) { + + UInt32 tempPrices[kNumFullDistances]; + unsigned i, lps; + + const CProbPrice *ProbPrices = p->ProbPrices; + p->matchPriceCount = 0; + + for (i = kStartPosModelIndex / 2; i < kNumFullDistances / 2; i++) + { + unsigned posSlot = GetPosSlot1(i); + unsigned footerBits = (posSlot >> 1) - 1; + unsigned base = ((2 | (posSlot & 1)) << footerBits); + const CLzmaProb *probs = p->posEncoders + (size_t)base * 2; + // tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base, footerBits, i - base, p->ProbPrices); + UInt32 price = 0; + unsigned m = 1; + unsigned sym = i; + unsigned offset = (unsigned)1 << footerBits; + base += i; + + if (footerBits) + do { - UInt32 posSlot = GetPosSlot1(i); - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base - posSlot - 1, footerBits, i - base, p->ProbPrices); + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[m], bit); + m = (m << 1) + bit; + } + while (--footerBits); + + { + unsigned prob = probs[m]; + tempPrices[base ] = price + GET_PRICEa_0(prob); + tempPrices[base + offset] = price + GET_PRICEa_1(prob); + } + } + + for (lps = 0; lps < kNumLenToPosStates; lps++) + { + unsigned slot; + unsigned distTableSize2 = (p->distTableSize + 1) >> 1; + UInt32 *posSlotPrices = p->posSlotPrices[lps]; + const CLzmaProb *probs = p->posSlotEncoder[lps]; + + for (slot = 0; slot < distTableSize2; slot++) + { + // posSlotPrices[slot] = RcTree_GetPrice(encoder, kNumPosSlotBits, slot, p->ProbPrices); + UInt32 price; + unsigned bit; + unsigned sym = slot + (1 << (kNumPosSlotBits - 1)); + unsigned prob; + bit = sym & 1; sym >>= 1; price = GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + prob = probs[(size_t)slot + (1 << (kNumPosSlotBits - 1))]; + posSlotPrices[(size_t)slot * 2 ] = price + GET_PRICEa_0(prob); + posSlotPrices[(size_t)slot * 2 + 1] = price + GET_PRICEa_1(prob); + } + + { + UInt32 delta = ((UInt32)((kEndPosModelIndex / 2 - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + for (slot = kEndPosModelIndex / 2; slot < distTableSize2; slot++) + { + posSlotPrices[(size_t)slot * 2 ] += delta; + posSlotPrices[(size_t)slot * 2 + 1] += delta; + delta += ((UInt32)1 << kNumBitPriceShiftBits); + } } - for (lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) { - UInt32 posSlot; - const CLzmaProb *encoder = p->posSlotEncoder[lenToPosState]; - UInt32 *posSlotPrices = p->posSlotPrices[lenToPosState]; - for (posSlot = 0; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] = RcTree_GetPrice(encoder, kNumPosSlotBits, posSlot, p->ProbPrices); - for (posSlot = kEndPosModelIndex; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] += ((((posSlot >> 1) - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + UInt32 *dp = p->distancesPrices[lps]; + + dp[0] = posSlotPrices[0]; + dp[1] = posSlotPrices[1]; + dp[2] = posSlotPrices[2]; + dp[3] = posSlotPrices[3]; + for (i = 4; i < kNumFullDistances; i += 2) + { + UInt32 slotPrice = posSlotPrices[GetPosSlot1(i)]; + dp[i ] = slotPrice + tempPrices[i]; + dp[i + 1] = slotPrice + tempPrices[i + 1]; + } + } + } + // } +} + + + +static void LzmaEnc_Construct(CLzmaEnc *p) +{ + RangeEnc_Construct(&p->rc); + MatchFinder_Construct(&MFB); + + #ifndef _7ZIP_ST + p->matchFinderMt.MatchFinder = &MFB; + MatchFinderMt_Construct(&p->matchFinderMt); + #endif + + { + CLzmaEncProps props; + LzmaEncProps_Init(&props); + LzmaEnc_SetProps(p, &props); + } + + #ifndef LZMA_LOG_BSR + LzmaEnc_FastPosInit(p->g_FastPos); + #endif + + LzmaEnc_InitPriceTables(p->ProbPrices); + p->litProbs = NULL; + p->saveState.litProbs = NULL; +} + +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc) +{ + void *p; + p = ISzAlloc_Alloc(alloc, sizeof(CLzmaEnc)); + if (p) + LzmaEnc_Construct((CLzmaEnc *)p); + return p; +} + +static void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->litProbs); + ISzAlloc_Free(alloc, p->saveState.litProbs); + p->litProbs = NULL; + p->saveState.litProbs = NULL; +} + +static void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + #ifndef _7ZIP_ST + MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); + #endif + + MatchFinder_Free(&MFB, allocBig); + LzmaEnc_FreeLits(p, alloc); + RangeEnc_Free(&p->rc, alloc); +} + +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + LzmaEnc_Destruct((CLzmaEnc *)p, alloc, allocBig); + ISzAlloc_Free(alloc, p); +} + + +MY_NO_INLINE +static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpackSize) +{ + UInt32 nowPos32, startPos32; + if (p->needInit) + { + #ifndef _7ZIP_ST + if (p->mtMode) + { + RINOK(MatchFinderMt_InitMt(&p->matchFinderMt)); + } + #endif + p->matchFinder.Init(p->matchFinderObj); + p->needInit = 0; + } + + if (p->finished) + return p->result; + RINOK(CheckErrors(p)); + + nowPos32 = (UInt32)p->nowPos64; + startPos32 = nowPos32; + + if (p->nowPos64 == 0) + { + unsigned numPairs; + Byte curByte; + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) + return Flush(p, nowPos32); + ReadMatchDistances(p, &numPairs); + RangeEnc_EncodeBit_0(&p->rc, &p->isMatch[kState_Start][0]); + // p->state = kLiteralNextStates[p->state]; + curByte = *(p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset); + LitEnc_Encode(&p->rc, p->litProbs, curByte); + p->additionalOffset--; + nowPos32++; + } + + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0) + + for (;;) + { + UInt32 dist; + unsigned len, posState; + UInt32 range, ttt, newBound; + CLzmaProb *probs; + + if (p->fastMode) + len = GetOptimumFast(p); + else + { + unsigned oci = p->optCur; + if (p->optEnd == oci) + len = GetOptimum(p, nowPos32); + else + { + const COptimal *opt = &p->opt[oci]; + len = opt->len; + p->backRes = opt->dist; + p->optCur = oci + 1; + } + } + + posState = (unsigned)nowPos32 & p->pbMask; + range = p->rc.range; + probs = &p->isMatch[p->state][posState]; + + RC_BIT_PRE(&p->rc, probs) + + dist = p->backRes; + + #ifdef SHOW_STAT2 + printf("\n pos = %6X, len = %3u pos = %6u", nowPos32, len, dist); + #endif + + if (dist == MARK_LIT) + { + Byte curByte; + const Byte *data; + unsigned state; + + RC_BIT_0(&p->rc, probs); + p->rc.range = range; + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; + probs = LIT_PROBS(nowPos32, *(data - 1)); + curByte = *data; + state = p->state; + p->state = kLiteralNextStates[state]; + if (IsLitState(state)) + LitEnc_Encode(&p->rc, probs, curByte); + else + LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0])); + } + else + { + RC_BIT_1(&p->rc, probs); + probs = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, probs) + + if (dist < LZMA_NUM_REPS) + { + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG0[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 0) { - UInt32 *distancesPrices = p->distancesPrices[lenToPosState]; - UInt32 i; - for (i = 0; i < kStartPosModelIndex; i++) - distancesPrices[i] = posSlotPrices[i]; - for (; i < kNumFullDistances; i++) - distancesPrices[i] = posSlotPrices[GetPosSlot1(i)] + tempPrices[i]; - } - } - p->matchPriceCount = 0; -} - -void LzmaEnc_Construct(CLzmaEnc *p) -{ - RangeEnc_Construct(&p->rc); - MatchFinder_Construct(&p->matchFinderBase); -#ifndef _7ZIP_ST - MatchFinderMt_Construct(&p->matchFinderMt); - p->matchFinderMt.MatchFinder = &p->matchFinderBase; -#endif - - { - CLzmaEncProps props; - LzmaEncProps_Init(&props); - LzmaEnc_SetProps(p, &props); - } - -#ifndef LZMA_LOG_BSR - LzmaEnc_FastPosInit(p->g_FastPos); -#endif - - LzmaEnc_InitPriceTables(p->ProbPrices); - p->litProbs = 0; - p->saveState.litProbs = 0; -} - -CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc) -{ - void *p; - p = alloc->Alloc(alloc, sizeof(CLzmaEnc)); - if (p != 0) - LzmaEnc_Construct((CLzmaEnc *)p); - return p; -} - -void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAlloc *alloc) -{ - alloc->Free(alloc, p->litProbs); - alloc->Free(alloc, p->saveState.litProbs); - p->litProbs = 0; - p->saveState.litProbs = 0; -} - -void LzmaEnc_Destruct(CLzmaEnc *p, ISzAlloc *alloc, ISzAlloc *allocBig) -{ -#ifndef _7ZIP_ST - MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); -#endif - MatchFinder_Free(&p->matchFinderBase, allocBig); - LzmaEnc_FreeLits(p, alloc); - RangeEnc_Free(&p->rc, alloc); -} - -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig) -{ - LzmaEnc_Destruct((CLzmaEnc *)p, alloc, allocBig); - alloc->Free(alloc, p); -} - -static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize, UInt32 maxUnpackSize) -{ - UInt32 nowPos32, startPos32; - if (p->needInit) - { - p->matchFinder.Init(p->matchFinderObj); - p->needInit = 0; - } - - if (p->finished) - return p->result; - RINOK(CheckErrors(p)); - - nowPos32 = (UInt32)p->nowPos64; - startPos32 = nowPos32; - - if (p->nowPos64 == 0) - { - UInt32 numPairs; - Byte curByte; - if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) - return Flush(p, nowPos32); - ReadMatchDistances(p, &numPairs); - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][0], 0); - p->state = kLiteralNextStates[p->state]; - curByte = p->matchFinder.GetIndexByte(p->matchFinderObj, 0 - p->additionalOffset); - LitEnc_Encode(&p->rc, p->litProbs, curByte); - p->additionalOffset--; - nowPos32++; - } - - if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0) - for (;;) - { - UInt32 pos, len, posState; - - if (p->fastMode) - len = GetOptimumFast(p, &pos); - else - len = GetOptimum(p, nowPos32, &pos); - -#ifdef SHOW_STAT2 - printf("\n pos = %4X, len = %d pos = %d", nowPos32, len, pos); -#endif - - posState = nowPos32 & p->pbMask; - if (len == 1 && pos == (UInt32)-1) - { - Byte curByte; - CLzmaProb *probs; - const Byte *data; - - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 0); - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; - curByte = *data; - probs = LIT_PROBS(nowPos32, *(data - 1)); - if (IsCharState(p->state)) - LitEnc_Encode(&p->rc, probs, curByte); - else - LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0] - 1)); - p->state = kLiteralNextStates[p->state]; + RC_BIT_0(&p->rc, probs); + probs = &p->isRep0Long[p->state][posState]; + RC_BIT_PRE(&p->rc, probs) + if (len != 1) + { + RC_BIT_1_BASE(&p->rc, probs); + } + else + { + RC_BIT_0_BASE(&p->rc, probs); + p->state = kShortRepNextStates[p->state]; + } } else { - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - if (pos < LZMA_NUM_REPS) + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG1[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 1) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[1]; + } + else + { + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG2[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 2) { - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 1); - if (pos == 0) - { - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 0); - RangeEnc_EncodeBit(&p->rc, &p->isRep0Long[p->state][posState], ((len == 1) ? 0 : 1)); - } - else - { - UInt32 distance = p->reps[pos]; - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 1); - if (pos == 1) - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 0); - else - { - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRepG2[p->state], pos - 2); - if (pos == 3) - p->reps[3] = p->reps[2]; - p->reps[2] = p->reps[1]; - } - p->reps[1] = p->reps[0]; - p->reps[0] = distance; - } - if (len == 1) - p->state = kShortRepNextStates[p->state]; - else - { - LenEnc_Encode2(&p->repLenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - p->state = kRepNextStates[p->state]; - } + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[2]; } else { - UInt32 posSlot; - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); - p->state = kMatchNextStates[p->state]; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - pos -= LZMA_NUM_REPS; - GetPosSlot(pos, posSlot); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, posSlot); - - if (posSlot >= kStartPosModelIndex) - { - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - UInt32 posReduced = pos - base; - - if (posSlot < kEndPosModelIndex) - RcTree_ReverseEncode(&p->rc, p->posEncoders + base - posSlot - 1, footerBits, posReduced); - else - { - RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); - p->alignPriceCount++; - } - } - p->reps[3] = p->reps[2]; - p->reps[2] = p->reps[1]; - p->reps[1] = p->reps[0]; - p->reps[0] = pos; - p->matchPriceCount++; + RC_BIT_1_BASE(&p->rc, probs); + dist = p->reps[3]; + p->reps[3] = p->reps[2]; } + p->reps[2] = p->reps[1]; + } + p->reps[1] = p->reps[0]; + p->reps[0] = dist; } - p->additionalOffset -= len; - nowPos32 += len; - if (p->additionalOffset == 0) + + RC_NORM(&p->rc) + + p->rc.range = range; + + if (len != 1) { - UInt32 processed; - if (!p->fastMode) - { - if (p->matchPriceCount >= (1 << 7)) - FillDistancesPrices(p); - if (p->alignPriceCount >= kAlignTableSize) - FillAlignPrices(p); - } - if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) - break; - processed = nowPos32 - startPos32; - if (useLimits) - { - if (processed + kNumOpts + 300 >= maxUnpackSize || - RangeEnc_GetProcessed(&p->rc) + kNumOpts * 2 >= maxPackSize) - break; - } - else if (processed >= (1 << 15)) - { - p->nowPos64 += nowPos32 - startPos32; - return CheckErrors(p); - } + LenEnc_Encode(&p->repLenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + --p->repLenEncCounter; + p->state = kRepNextStates[p->state]; } + } + else + { + unsigned posSlot; + RC_BIT_0(&p->rc, probs); + p->rc.range = range; + p->state = kMatchNextStates[p->state]; + + LenEnc_Encode(&p->lenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + // --p->lenEnc.counter; + + dist -= LZMA_NUM_REPS; + p->reps[3] = p->reps[2]; + p->reps[2] = p->reps[1]; + p->reps[1] = p->reps[0]; + p->reps[0] = dist + 1; + + p->matchPriceCount++; + GetPosSlot(dist, posSlot); + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], posSlot); + { + UInt32 sym = (UInt32)posSlot + (1 << kNumPosSlotBits); + range = p->rc.range; + probs = p->posSlotEncoder[GetLenToPosState(len)]; + do + { + CLzmaProb *prob = probs + (sym >> kNumPosSlotBits); + UInt32 bit = (sym >> (kNumPosSlotBits - 1)) & 1; + sym <<= 1; + RC_BIT(&p->rc, prob, bit); + } + while (sym < (1 << kNumPosSlotBits * 2)); + p->rc.range = range; } - p->nowPos64 += nowPos32 - startPos32; - return Flush(p, nowPos32); + + if (dist >= kStartPosModelIndex) + { + unsigned footerBits = ((posSlot >> 1) - 1); + + if (dist < kNumFullDistances) + { + unsigned base = ((2 | (posSlot & 1)) << footerBits); + RcTree_ReverseEncode(&p->rc, p->posEncoders + base, footerBits, (unsigned)(dist /* - base */)); + } + else + { + UInt32 pos2 = (dist | 0xF) << (32 - footerBits); + range = p->rc.range; + // RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); + /* + do + { + range >>= 1; + p->rc.low += range & (0 - ((dist >> --footerBits) & 1)); + RC_NORM(&p->rc) + } + while (footerBits > kNumAlignBits); + */ + do + { + range >>= 1; + p->rc.low += range & (0 - (pos2 >> 31)); + pos2 += pos2; + RC_NORM(&p->rc) + } + while (pos2 != 0xF0000000); + + + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); + + { + unsigned m = 1; + unsigned bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); + p->rc.range = range; + // p->alignPriceCount++; + } + } + } + } + } + + nowPos32 += (UInt32)len; + p->additionalOffset -= len; + + if (p->additionalOffset == 0) + { + UInt32 processed; + + if (!p->fastMode) + { + /* + if (p->alignPriceCount >= 16) // kAlignTableSize + FillAlignPrices(p); + if (p->matchPriceCount >= 128) + FillDistancesPrices(p); + if (p->lenEnc.counter <= 0) + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + */ + if (p->matchPriceCount >= 64) + { + FillAlignPrices(p); + // { int y; for (y = 0; y < 100; y++) { + FillDistancesPrices(p); + // }} + LenPriceEnc_UpdateTables(&p->lenEnc, (unsigned)1 << p->pb, &p->lenProbs, p->ProbPrices); + } + if (p->repLenEncCounter <= 0) + { + p->repLenEncCounter = REP_LEN_COUNT; + LenPriceEnc_UpdateTables(&p->repLenEnc, (unsigned)1 << p->pb, &p->repLenProbs, p->ProbPrices); + } + } + + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) + break; + processed = nowPos32 - startPos32; + + if (maxPackSize) + { + if (processed + kNumOpts + 300 >= maxUnpackSize + || RangeEnc_GetProcessed_sizet(&p->rc) + kPackReserve >= maxPackSize) + break; + } + else if (processed >= (1 << 17)) + { + p->nowPos64 += nowPos32 - startPos32; + return CheckErrors(p); + } + } + } + + p->nowPos64 += nowPos32 - startPos32; + return Flush(p, nowPos32); } + + #define kBigHashDicLimit ((UInt32)1 << 24) -static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) +static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - UInt32 beforeSize = kNumOpts; - if (!RangeEnc_Alloc(&p->rc, alloc)) - return SZ_ERROR_MEM; + UInt32 beforeSize = kNumOpts; + UInt32 dictSize; -#ifndef _7ZIP_ST - { - Bool btMode = (p->matchFinderBase.btMode != 0); - p->mtMode = (p->multiThread && !p->fastMode && btMode); - } -#endif + if (!RangeEnc_Alloc(&p->rc, alloc)) + return SZ_ERROR_MEM; - { - unsigned lclp = p->lc + p->lp; - if (p->litProbs == 0 || p->saveState.litProbs == 0 || p->lclp != lclp) - { - LzmaEnc_FreeLits(p, alloc); - p->litProbs = (CLzmaProb *)alloc->Alloc(alloc, (0x300 << lclp) * sizeof(CLzmaProb)); - p->saveState.litProbs = (CLzmaProb *)alloc->Alloc(alloc, (0x300 << lclp) * sizeof(CLzmaProb)); - if (p->litProbs == 0 || p->saveState.litProbs == 0) - { - LzmaEnc_FreeLits(p, alloc); - return SZ_ERROR_MEM; - } - p->lclp = lclp; - } - } - - p->matchFinderBase.bigHash = (p->dictSize > kBigHashDicLimit); - - if (beforeSize + p->dictSize < keepWindowSize) - beforeSize = keepWindowSize - p->dictSize; - -#ifndef _7ZIP_ST - if (p->mtMode) - { - RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)); - p->matchFinderObj = &p->matchFinderMt; - MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); - } - else -#endif - { - if (!MatchFinder_Create(&p->matchFinderBase, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)) - return SZ_ERROR_MEM; - p->matchFinderObj = &p->matchFinderBase; - MatchFinder_CreateVTable(&p->matchFinderBase, &p->matchFinder); - } - return SZ_OK; -} - -void LzmaEnc_Init(CLzmaEnc *p) -{ - UInt32 i; - p->state = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - p->reps[i] = 0; - - RangeEnc_Init(&p->rc); - - for (i = 0; i < kNumStates; i++) - { - UInt32 j; - for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) - { - p->isMatch[i][j] = kProbInitValue; - p->isRep0Long[i][j] = kProbInitValue; - } - p->isRep[i] = kProbInitValue; - p->isRepG0[i] = kProbInitValue; - p->isRepG1[i] = kProbInitValue; - p->isRepG2[i] = kProbInitValue; - } + #ifndef _7ZIP_ST + p->mtMode = (p->multiThread && !p->fastMode && (MFB.btMode != 0)); + #endif { - UInt32 num = 0x300 << (p->lp + p->lc); - for (i = 0; i < num; i++) - p->litProbs[i] = kProbInitValue; - } - - { - for (i = 0; i < kNumLenToPosStates; i++) + unsigned lclp = p->lc + p->lp; + if (!p->litProbs || !p->saveState.litProbs || p->lclp != lclp) + { + LzmaEnc_FreeLits(p, alloc); + p->litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + p->saveState.litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + if (!p->litProbs || !p->saveState.litProbs) { - CLzmaProb *probs = p->posSlotEncoder[i]; - UInt32 j; - for (j = 0; j < (1 << kNumPosSlotBits); j++) - probs[j] = kProbInitValue; + LzmaEnc_FreeLits(p, alloc); + return SZ_ERROR_MEM; } + p->lclp = lclp; + } + } + + MFB.bigHash = (Byte)(p->dictSize > kBigHashDicLimit ? 1 : 0); + + + dictSize = p->dictSize; + if (dictSize == ((UInt32)2 << 30) || + dictSize == ((UInt32)3 << 30)) + { + /* 21.03 : here we reduce the dictionary for 2 reasons: + 1) we don't want 32-bit back_distance matches in decoder for 2 GB dictionary. + 2) we want to elimate useless last MatchFinder_Normalize3() for corner cases, + where data size is aligned for 1 GB: 5/6/8 GB. + That reducing must be >= 1 for such corner cases. */ + dictSize -= 1; + } + + if (beforeSize + dictSize < keepWindowSize) + beforeSize = keepWindowSize - dictSize; + + /* in worst case we can look ahead for + max(LZMA_MATCH_LEN_MAX, numFastBytes + 1 + numFastBytes) bytes. + we send larger value for (keepAfter) to MantchFinder_Create(): + (numFastBytes + LZMA_MATCH_LEN_MAX + 1) + */ + + #ifndef _7ZIP_ST + if (p->mtMode) + { + RINOK(MatchFinderMt_Create(&p->matchFinderMt, dictSize, beforeSize, + p->numFastBytes, LZMA_MATCH_LEN_MAX + 1 /* 18.04 */ + , allocBig)); + p->matchFinderObj = &p->matchFinderMt; + MFB.bigHash = (Byte)( + (p->dictSize > kBigHashDicLimit && MFB.hashMask >= 0xFFFFFF) ? 1 : 0); + MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); + } + else + #endif + { + if (!MatchFinder_Create(&MFB, dictSize, beforeSize, + p->numFastBytes, LZMA_MATCH_LEN_MAX + 1 /* 21.03 */ + , allocBig)) + return SZ_ERROR_MEM; + p->matchFinderObj = &MFB; + MatchFinder_CreateVTable(&MFB, &p->matchFinder); + } + + return SZ_OK; +} + +static void LzmaEnc_Init(CLzmaEnc *p) +{ + unsigned i; + p->state = 0; + p->reps[0] = + p->reps[1] = + p->reps[2] = + p->reps[3] = 1; + + RangeEnc_Init(&p->rc); + + for (i = 0; i < (1 << kNumAlignBits); i++) + p->posAlignEncoder[i] = kProbInitValue; + + for (i = 0; i < kNumStates; i++) + { + unsigned j; + for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) + { + p->isMatch[i][j] = kProbInitValue; + p->isRep0Long[i][j] = kProbInitValue; + } + p->isRep[i] = kProbInitValue; + p->isRepG0[i] = kProbInitValue; + p->isRepG1[i] = kProbInitValue; + p->isRepG2[i] = kProbInitValue; + } + + { + for (i = 0; i < kNumLenToPosStates; i++) + { + CLzmaProb *probs = p->posSlotEncoder[i]; + unsigned j; + for (j = 0; j < (1 << kNumPosSlotBits); j++) + probs[j] = kProbInitValue; + } } { - for (i = 0; i < kNumFullDistances - kEndPosModelIndex; i++) - p->posEncoders[i] = kProbInitValue; + for (i = 0; i < kNumFullDistances; i++) + p->posEncoders[i] = kProbInitValue; } - LenEnc_Init(&p->lenEnc.p); - LenEnc_Init(&p->repLenEnc.p); + { + UInt32 num = (UInt32)0x300 << (p->lp + p->lc); + UInt32 k; + CLzmaProb *probs = p->litProbs; + for (k = 0; k < num; k++) + probs[k] = kProbInitValue; + } - for (i = 0; i < (1 << kNumAlignBits); i++) - p->posAlignEncoder[i] = kProbInitValue; - p->optimumEndIndex = 0; - p->optimumCurrentIndex = 0; - p->additionalOffset = 0; + LenEnc_Init(&p->lenProbs); + LenEnc_Init(&p->repLenProbs); - p->pbMask = (1 << p->pb) - 1; - p->lpMask = (1 << p->lp) - 1; + p->optEnd = 0; + p->optCur = 0; + + { + for (i = 0; i < kNumOpts; i++) + p->opt[i].price = kInfinityPrice; + } + + p->additionalOffset = 0; + + p->pbMask = ((unsigned)1 << p->pb) - 1; + p->lpMask = ((UInt32)0x100 << p->lp) - ((unsigned)0x100 >> p->lc); + + // p->mf_Failure = False; } -void LzmaEnc_InitPrices(CLzmaEnc *p) -{ - if (!p->fastMode) - { - FillDistancesPrices(p); - FillAlignPrices(p); - } - p->lenEnc.tableSize = - p->repLenEnc.tableSize = - p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, p->ProbPrices); - LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, p->ProbPrices); +static void LzmaEnc_InitPrices(CLzmaEnc *p) +{ + if (!p->fastMode) + { + FillDistancesPrices(p); + FillAlignPrices(p); + } + + p->lenEnc.tableSize = + p->repLenEnc.tableSize = + p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; + + p->repLenEncCounter = REP_LEN_COUNT; + + LenPriceEnc_UpdateTables(&p->lenEnc, (unsigned)1 << p->pb, &p->lenProbs, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->repLenEnc, (unsigned)1 << p->pb, &p->repLenProbs, p->ProbPrices); } -static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) +static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - UInt32 i; - for (i = 0; i < (UInt32)kDicLogSizeMaxCompress; i++) - if (p->dictSize <= ((UInt32)1 << i)) - break; - p->distTableSize = i * 2; + unsigned i; + for (i = kEndPosModelIndex / 2; i < kDicLogSizeMax; i++) + if (p->dictSize <= ((UInt32)1 << i)) + break; + p->distTableSize = i * 2; - p->finished = False; - p->result = SZ_OK; - RINOK(LzmaEnc_Alloc(p, keepWindowSize, alloc, allocBig)); - LzmaEnc_Init(p); - LzmaEnc_InitPrices(p); - p->nowPos64 = 0; - return SZ_OK; + p->finished = False; + p->result = SZ_OK; + RINOK(LzmaEnc_Alloc(p, keepWindowSize, alloc, allocBig)); + LzmaEnc_Init(p); + LzmaEnc_InitPrices(p); + p->nowPos64 = 0; + return SZ_OK; } static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { - CLzmaEnc *p = (CLzmaEnc *)pp; - p->matchFinderBase.stream = inStream; - p->needInit = 1; - p->rc.outStream = outStream; - return LzmaEnc_AllocAndInit(p, 0, alloc, allocBig); + CLzmaEnc *p = (CLzmaEnc *)pp; + MFB.stream = inStream; + p->needInit = 1; + p->rc.outStream = outStream; + return LzmaEnc_AllocAndInit(p, 0, alloc, allocBig); } SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { - CLzmaEnc *p = (CLzmaEnc *)pp; - p->matchFinderBase.stream = inStream; - p->needInit = 1; - return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); + CLzmaEnc *p = (CLzmaEnc *)pp; + MFB.stream = inStream; + p->needInit = 1; + return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); } static void LzmaEnc_SetInputBuf(CLzmaEnc *p, const Byte *src, SizeT srcLen) { - p->matchFinderBase.directInput = 1; - p->matchFinderBase.bufferBase = (Byte *)src; - p->matchFinderBase.directInputRem = srcLen; + MFB.directInput = 1; + MFB.bufferBase = (Byte *)src; + MFB.directInputRem = srcLen; } SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, - UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - CLzmaEnc *p = (CLzmaEnc *)pp; - LzmaEnc_SetInputBuf(p, src, srcLen); - p->needInit = 1; + CLzmaEnc *p = (CLzmaEnc *)pp; + LzmaEnc_SetInputBuf(p, src, srcLen); + p->needInit = 1; - return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); + LzmaEnc_SetDataSize(pp, srcLen); + return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); } void LzmaEnc_Finish(CLzmaEncHandle pp) { -#ifndef _7ZIP_ST - CLzmaEnc *p = (CLzmaEnc *)pp; - if (p->mtMode) - MatchFinderMt_ReleaseStream(&p->matchFinderMt); -#else - (void)pp; -#endif + #ifndef _7ZIP_ST + CLzmaEnc *p = (CLzmaEnc *)pp; + if (p->mtMode) + MatchFinderMt_ReleaseStream(&p->matchFinderMt); + #else + UNUSED_VAR(pp); + #endif } + typedef struct { - ISeqOutStream funcTable; - Byte *data; - SizeT rem; - Bool overflow; -} CSeqOutStreamBuf; + ISeqOutStream vt; + Byte *data; + SizeT rem; + BoolInt overflow; +} CLzmaEnc_SeqOutStreamBuf; -static size_t MyWrite(void *pp, const void *data, size_t size) +static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size) { - CSeqOutStreamBuf *p = (CSeqOutStreamBuf *)pp; - if (p->rem < size) - { - size = p->rem; - p->overflow = True; - } + CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt); + if (p->rem < size) + { + size = p->rem; + p->overflow = True; + } + if (size != 0) + { memcpy(p->data, data, size); p->rem -= size; p->data += size; - return size; + } + return size; } + +/* UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp) { - const CLzmaEnc *p = (CLzmaEnc *)pp; - return p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); + const CLzmaEnc *p = (CLzmaEnc *)pp; + return p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); } +*/ const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) { - const CLzmaEnc *p = (CLzmaEnc *)pp; - return p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; + const CLzmaEnc *p = (CLzmaEnc *)pp; + return p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; } -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, + +// (desiredPackSize == 0) is not allowed +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize) { - CLzmaEnc *p = (CLzmaEnc *)pp; - UInt64 nowPos64; - SRes res; - CSeqOutStreamBuf outStream; + CLzmaEnc *p = (CLzmaEnc *)pp; + UInt64 nowPos64; + SRes res; + CLzmaEnc_SeqOutStreamBuf outStream; - outStream.funcTable.Write = MyWrite; - outStream.data = dest; - outStream.rem = *destLen; - outStream.overflow = False; + outStream.vt.Write = SeqOutStreamBuf_Write; + outStream.data = dest; + outStream.rem = *destLen; + outStream.overflow = False; - p->writeEndMark = False; - p->finished = False; - p->result = SZ_OK; + p->writeEndMark = False; + p->finished = False; + p->result = SZ_OK; - if (reInit) - LzmaEnc_Init(p); - LzmaEnc_InitPrices(p); - nowPos64 = p->nowPos64; - RangeEnc_Init(&p->rc); - p->rc.outStream = &outStream.funcTable; + if (reInit) + LzmaEnc_Init(p); + LzmaEnc_InitPrices(p); + RangeEnc_Init(&p->rc); + p->rc.outStream = &outStream.vt; + nowPos64 = p->nowPos64; + + res = LzmaEnc_CodeOneBlock(p, desiredPackSize, *unpackSize); + + *unpackSize = (UInt32)(p->nowPos64 - nowPos64); + *destLen -= outStream.rem; + if (outStream.overflow) + return SZ_ERROR_OUTPUT_EOF; - res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize); - - *unpackSize = (UInt32)(p->nowPos64 - nowPos64); - *destLen -= outStream.rem; - if (outStream.overflow) - return SZ_ERROR_OUTPUT_EOF; - - return res; + return res; } + +MY_NO_INLINE static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) { - SRes res = SZ_OK; + SRes res = SZ_OK; -#ifndef _7ZIP_ST - Byte allocaDummy[0x300]; - int i = 0; - for (i = 0; i < 16; i++) - allocaDummy[i] = (Byte)i; -#endif + #ifndef _7ZIP_ST + Byte allocaDummy[0x300]; + allocaDummy[0] = 0; + allocaDummy[1] = allocaDummy[0]; + #endif - for (;;) + for (;;) + { + res = LzmaEnc_CodeOneBlock(p, 0, 0); + if (res != SZ_OK || p->finished) + break; + if (progress) { - res = LzmaEnc_CodeOneBlock(p, False, 0, 0); - if (res != SZ_OK || p->finished != 0) - break; - if (progress != 0) - { - res = progress->Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); - if (res != SZ_OK) - { - res = SZ_ERROR_PROGRESS; - break; - } - } + res = ICompressProgress_Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); + if (res != SZ_OK) + { + res = SZ_ERROR_PROGRESS; + break; + } } - LzmaEnc_Finish(p); - return res; + } + + LzmaEnc_Finish(p); + + /* + if (res == SZ_OK && !Inline_MatchFinder_IsFinishedOK(&MFB)) + res = SZ_ERROR_FAIL; + } + */ + + return res; } + SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { - RINOK(LzmaEnc_Prepare(pp, outStream, inStream, alloc, allocBig)); - return LzmaEnc_Encode2((CLzmaEnc *)pp, progress); + RINOK(LzmaEnc_Prepare(pp, outStream, inStream, alloc, allocBig)); + return LzmaEnc_Encode2((CLzmaEnc *)pp, progress); } + SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) { - CLzmaEnc *p = (CLzmaEnc *)pp; - int i; - UInt32 dictSize = p->dictSize; - if (*size < LZMA_PROPS_SIZE) - return SZ_ERROR_PARAM; - *size = LZMA_PROPS_SIZE; + if (*size < LZMA_PROPS_SIZE) + return SZ_ERROR_PARAM; + *size = LZMA_PROPS_SIZE; + { + const CLzmaEnc *p = (const CLzmaEnc *)pp; + const UInt32 dictSize = p->dictSize; + UInt32 v; props[0] = (Byte)((p->pb * 5 + p->lp) * 9 + p->lc); - - for (i = 11; i <= 30; i++) + + // we write aligned dictionary value to properties for lzma decoder + if (dictSize >= ((UInt32)1 << 21)) { - if (dictSize <= ((UInt32)2 << i)) - { - dictSize = (2 << i); - break; - } - if (dictSize <= ((UInt32)3 << i)) - { - dictSize = (3 << i); - break; - } + const UInt32 kDictMask = ((UInt32)1 << 20) - 1; + v = (dictSize + kDictMask) & ~kDictMask; + if (v < dictSize) + v = dictSize; + } + else + { + unsigned i = 11 * 2; + do + { + v = (UInt32)(2 + (i & 1)) << (i >> 1); + i++; + } + while (v < dictSize); } - for (i = 0; i < 4; i++) - props[1 + i] = (Byte)(dictSize >> (8 * i)); + SetUi32(props + 1, v); return SZ_OK; + } } + +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle pp) +{ + return (unsigned)((CLzmaEnc *)pp)->writeEndMark; +} + + SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig) + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - SRes res; - CLzmaEnc *p = (CLzmaEnc *)pp; + SRes res; + CLzmaEnc *p = (CLzmaEnc *)pp; - CSeqOutStreamBuf outStream; + CLzmaEnc_SeqOutStreamBuf outStream; - LzmaEnc_SetInputBuf(p, src, srcLen); + outStream.vt.Write = SeqOutStreamBuf_Write; + outStream.data = dest; + outStream.rem = *destLen; + outStream.overflow = False; - outStream.funcTable.Write = MyWrite; - outStream.data = dest; - outStream.rem = *destLen; - outStream.overflow = False; + p->writeEndMark = writeEndMark; + p->rc.outStream = &outStream.vt; - p->writeEndMark = writeEndMark; + res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig); + + if (res == SZ_OK) + { + res = LzmaEnc_Encode2(p, progress); + if (res == SZ_OK && p->nowPos64 != srcLen) + res = SZ_ERROR_FAIL; + } - p->rc.outStream = &outStream.funcTable; - res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig); - if (res == SZ_OK) - res = LzmaEnc_Encode2(p, progress); - - *destLen -= outStream.rem; - if (outStream.overflow) - return SZ_ERROR_OUTPUT_EOF; - return res; + *destLen -= outStream.rem; + if (outStream.overflow) + return SZ_ERROR_OUTPUT_EOF; + return res; } + SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig) + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - CLzmaEnc *p = (CLzmaEnc *)LzmaEnc_Create(alloc); - SRes res; - if (p == 0) - return SZ_ERROR_MEM; + CLzmaEnc *p = (CLzmaEnc *)LzmaEnc_Create(alloc); + SRes res; + if (!p) + return SZ_ERROR_MEM; - res = LzmaEnc_SetProps(p, props); + res = LzmaEnc_SetProps(p, props); + if (res == SZ_OK) + { + res = LzmaEnc_WriteProperties(p, propsEncoded, propsSize); if (res == SZ_OK) - { - res = LzmaEnc_WriteProperties(p, propsEncoded, propsSize); - if (res == SZ_OK) - res = LzmaEnc_MemEncode(p, dest, destLen, src, srcLen, - writeEndMark, progress, alloc, allocBig); - } + res = LzmaEnc_MemEncode(p, dest, destLen, src, srcLen, + writeEndMark, progress, alloc, allocBig); + } - LzmaEnc_Destroy(p, alloc, allocBig); - return res; + LzmaEnc_Destroy(p, alloc, allocBig); + return res; } + + +/* +#ifndef _7ZIP_ST +void LzmaEnc_GetLzThreads(CLzmaEncHandle pp, HANDLE lz_threads[2]) +{ + const CLzmaEnc *p = (CLzmaEnc *)pp; + lz_threads[0] = p->matchFinderMt.hashSync.thread; + lz_threads[1] = p->matchFinderMt.btSync.thread; +} +#endif +*/ diff --git a/common/LZMA/SDK/C/LzmaEnc.h b/common/LZMA/SDK/C/LzmaEnc.h index 200d60e..bc2ed50 100644 --- a/common/LZMA/SDK/C/LzmaEnc.h +++ b/common/LZMA/SDK/C/LzmaEnc.h @@ -1,23 +1,21 @@ /* LzmaEnc.h -- LZMA Encoder -2009-02-07 : Igor Pavlov : Public domain */ +2019-10-30 : Igor Pavlov : Public domain */ #ifndef __LZMA_ENC_H #define __LZMA_ENC_H -#include "Types.h" +#include "7zTypes.h" -#ifdef __cplusplus -extern "C" { -#endif +EXTERN_C_BEGIN #define LZMA_PROPS_SIZE 5 typedef struct _CLzmaEncProps { - int level; /* 0 <= level <= 9 */ + int level; /* 0 <= level <= 9 */ UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version - (1 << 12) <= dictSize <= (1 << 30) for 64-bit version - default = (1 << 24) */ + (1 << 12) <= dictSize <= (3 << 29) for 64-bit version + default = (1 << 24) */ int lc; /* 0 <= lc <= 8, default = 3 */ int lp; /* 0 <= lp <= 4, default = 0 */ int pb; /* 0 <= pb <= 4, default = 2 */ @@ -25,9 +23,14 @@ typedef struct _CLzmaEncProps int fb; /* 5 <= fb <= 273, default = 32 */ int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */ int numHashBytes; /* 2, 3 or 4, default = 4 */ - UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ + UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */ int numThreads; /* 1 or 2, default = 2 */ + + UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1. + Encoder uses this value to reduce dictionary size */ + + UInt64 affinity; } CLzmaEncProps; void LzmaEncProps_Init(CLzmaEncProps *p); @@ -37,44 +40,39 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2); /* ---------- CLzmaEncHandle Interface ---------- */ -/* LzmaEnc_* functions can return the following exit codes: -Returns: +/* LzmaEnc* functions can return the following exit codes: +SRes: SZ_OK - OK SZ_ERROR_MEM - Memory allocation error SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - Write callback error. + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ typedef void * CLzmaEncHandle; -CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc); -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig); +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc); +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig); + SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props); +void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize); SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size); +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p); + SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); + /* ---------- One Call Interface ---------- */ -/* LzmaEncode -Return code: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ - SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); -#ifdef __cplusplus -} -#endif +EXTERN_C_END #endif diff --git a/common/LZMA/SDK/C/Precomp.h b/common/LZMA/SDK/C/Precomp.h new file mode 100644 index 0000000..fc663b6 --- /dev/null +++ b/common/LZMA/SDK/C/Precomp.h @@ -0,0 +1,12 @@ +/* Precomp.h -- StdAfx +2013-11-12 : Igor Pavlov : Public domain */ + +#ifndef __7Z_PRECOMP_H +#define __7Z_PRECOMP_H + +#include "Compiler.h" +/* #include "7zTypes.h" */ + +#define _7ZIP_ST + +#endif diff --git a/common/LZMA/SDK/C/Types.h b/common/LZMA/SDK/C/Types.h deleted file mode 100644 index 90f6752..0000000 --- a/common/LZMA/SDK/C/Types.h +++ /dev/null @@ -1,256 +0,0 @@ -/* Types.h -- Basic types -2010-10-09 : Igor Pavlov : Public domain */ - -#ifndef __7Z_TYPES_H -#define __7Z_TYPES_H - -#include "../../UefiLzma.h" - -#include - -#ifdef _WIN32 -#include -#endif - -#ifndef EXTERN_C_BEGIN -#ifdef __cplusplus -#define EXTERN_C_BEGIN extern "C" { -#define EXTERN_C_END } -#else -#define EXTERN_C_BEGIN -#define EXTERN_C_END -#endif -#endif - -EXTERN_C_BEGIN - -#define SZ_OK 0 - -#define SZ_ERROR_DATA 1 -#define SZ_ERROR_MEM 2 -#define SZ_ERROR_CRC 3 -#define SZ_ERROR_UNSUPPORTED 4 -#define SZ_ERROR_PARAM 5 -#define SZ_ERROR_INPUT_EOF 6 -#define SZ_ERROR_OUTPUT_EOF 7 -#define SZ_ERROR_READ 8 -#define SZ_ERROR_WRITE 9 -#define SZ_ERROR_PROGRESS 10 -#define SZ_ERROR_FAIL 11 -#define SZ_ERROR_THREAD 12 - -#define SZ_ERROR_ARCHIVE 16 -#define SZ_ERROR_NO_ARCHIVE 17 - -typedef int SRes; - -#ifdef _WIN32 -typedef DWORD WRes; -#else -typedef int WRes; -#endif - -#ifndef RINOK -#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; } -#endif - -typedef unsigned char Byte; -typedef short Int16; -typedef unsigned short UInt16; - -#ifdef _LZMA_UINT32_IS_ULONG -typedef long Int32; -typedef unsigned long UInt32; -#else -typedef int Int32; -typedef unsigned int UInt32; -#endif - -#ifdef _SZ_NO_INT_64 - -/* define _SZ_NO_INT_64, if your compiler doesn't support 64-bit integers. - NOTES: Some code will work incorrectly in that case! */ - -typedef long Int64; -typedef unsigned long UInt64; - -#else - -#if defined(_MSC_VER) || defined(__BORLANDC__) -typedef __int64 Int64; -typedef unsigned __int64 UInt64; -#define UINT64_CONST(n) n -#else -typedef long long int Int64; -typedef unsigned long long int UInt64; -#define UINT64_CONST(n) n ## ULL -#endif - -#endif - -#ifdef _LZMA_NO_SYSTEM_SIZE_T -typedef UInt32 SizeT; -#else -typedef size_t SizeT; -#endif - -typedef int Bool; -#define True 1 -#define False 0 - - -#ifdef _WIN32 -#define MY_STD_CALL __stdcall -#else -#define MY_STD_CALL -#endif - -#ifdef _MSC_VER - -#if _MSC_VER >= 1300 -#define MY_NO_INLINE __declspec(noinline) -#else -#define MY_NO_INLINE -#endif - -#define MY_CDECL __cdecl -#define MY_FAST_CALL __fastcall - -#else - -#define MY_CDECL -#define MY_FAST_CALL - -#endif - - -/* The following interfaces use first parameter as pointer to structure */ - -typedef struct -{ - Byte (*Read)(void *p); /* reads one byte, returns 0 in case of EOF or error */ -} IByteIn; - -typedef struct -{ - void (*Write)(void *p, Byte b); -} IByteOut; - -typedef struct -{ - SRes (*Read)(void *p, void *buf, size_t *size); - /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. - (output(*size) < input(*size)) is allowed */ -} ISeqInStream; - -/* it can return SZ_ERROR_INPUT_EOF */ -SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size); -SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType); -SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf); - -typedef struct -{ - size_t (*Write)(void *p, const void *buf, size_t size); - /* Returns: result - the number of actually written bytes. - (result < size) means error */ -} ISeqOutStream; - -typedef enum -{ - SZ_SEEK_SET = 0, - SZ_SEEK_CUR = 1, - SZ_SEEK_END = 2 -} ESzSeek; - -typedef struct -{ - SRes (*Read)(void *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ - SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); -} ISeekInStream; - -typedef struct -{ - SRes (*Look)(void *p, const void **buf, size_t *size); - /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. - (output(*size) > input(*size)) is not allowed - (output(*size) < input(*size)) is allowed */ - SRes (*Skip)(void *p, size_t offset); - /* offset must be <= output(*size) of Look */ - - SRes (*Read)(void *p, void *buf, size_t *size); - /* reads directly (without buffer). It's same as ISeqInStream::Read */ - SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); -} ILookInStream; - -SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size); -SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset); - -/* reads via ILookInStream::Read */ -SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType); -SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size); - -#define LookToRead_BUF_SIZE (1 << 14) - -typedef struct -{ - ILookInStream s; - ISeekInStream *realStream; - size_t pos; - size_t size; - Byte buf[LookToRead_BUF_SIZE]; -} CLookToRead; - -void LookToRead_CreateVTable(CLookToRead *p, int lookahead); -void LookToRead_Init(CLookToRead *p); - -typedef struct -{ - ISeqInStream s; - ILookInStream *realStream; -} CSecToLook; - -void SecToLook_CreateVTable(CSecToLook *p); - -typedef struct -{ - ISeqInStream s; - ILookInStream *realStream; -} CSecToRead; - -void SecToRead_CreateVTable(CSecToRead *p); - -typedef struct -{ - SRes (*Progress)(void *p, UInt64 inSize, UInt64 outSize); - /* Returns: result. (result != SZ_OK) means break. - Value (UInt64)(Int64)-1 for size means unknown value. */ -} ICompressProgress; - -typedef struct -{ - void *(*Alloc)(void *p, size_t size); - void (*Free)(void *p, void *address); /* address can be 0 */ -} ISzAlloc; - -#define IAlloc_Alloc(p, size) (p)->Alloc((p), size) -#define IAlloc_Free(p, a) (p)->Free((p), a) - -#ifdef _WIN32 - -#define CHAR_PATH_SEPARATOR '\\' -#define WCHAR_PATH_SEPARATOR L'\\' -#define STRING_PATH_SEPARATOR "\\" -#define WSTRING_PATH_SEPARATOR L"\\" - -#else - -#define CHAR_PATH_SEPARATOR '/' -#define WCHAR_PATH_SEPARATOR L'/' -#define STRING_PATH_SEPARATOR "/" -#define WSTRING_PATH_SEPARATOR L"/" - -#endif - -EXTERN_C_END - -#endif diff --git a/common/basetypes.h b/common/basetypes.h index c6d7128..7b56bb9 100644 --- a/common/basetypes.h +++ b/common/basetypes.h @@ -64,8 +64,8 @@ typedef size_t USTATUS; #define U_INVALID_FIT 42 #define U_INVALID_MICROCODE 43 #define U_INVALID_ACM 44 -#define U_INVALID_BG_KEY_MANIFEST 45 -#define U_INVALID_BG_BOOT_POLICY 46 +#define U_INVALID_BOOT_GUARD_KEY_MANIFEST 45 +#define U_INVALID_BOOT_GUARD_BOOT_POLICY 46 #define U_INVALID_TXT_CONF 47 #define U_ELEMENTS_NOT_FOUND 48 #define U_PEI_CORE_ENTRY_POINT_NOT_FOUND 49 @@ -203,7 +203,30 @@ typedef struct EFI_TIME_ { #include #define ASSERT(x) assert(x) -// SHA256 hash size in bytes +// Hash sizes in bytes +#define SHA1_HASH_SIZE 0x14 #define SHA256_HASH_SIZE 0x20 +#define SHA384_HASH_SIZE 0x30 +#define SHA512_HASH_SIZE 0x40 +#define SM3_HASH_SIZE 0x20 + +// TCG Algorithm Registry: Table 2 +#define TCG_HASH_ALGORITHM_ID_SHA1 0x0004 +#define TCG_HASH_ALGORITHM_ID_SHA256 0x000B +#define TCG_HASH_ALGORITHM_ID_SHA384 0x000C +#define TCG_HASH_ALGORITHM_ID_SHA512 0x000D +#define TCG_HASH_ALGORITHM_ID_NULL 0x0010 +#define TCG_HASH_ALGORITHM_ID_SM3 0x0012 + +// A workaround for compilers not supporting c++11 and c11 +// for using PRIX64. +#define __STDC_FORMAT_MACROS +#include + +#if defined(__clang__) || defined(__GNUC__) +#define ATTRIBUTE_FORMAT_(t,f,a) __attribute__((format(t, f, a))) +#else +#define ATTRIBUTE_FORMAT_(t,f,a) +#endif #endif // BASETYPES_H diff --git a/common/bootguard.h b/common/bootguard.h deleted file mode 100644 index 533e6e7..0000000 --- a/common/bootguard.h +++ /dev/null @@ -1,210 +0,0 @@ -/* bootguard.h - -Copyright (c) 2017, LongSoft. All rights reserved. -This program and the accompanying materials -are licensed and made available under the terms and conditions of the BSD License -which accompanies this distribution. The full text of the license may be found at -http://opensource.org/licenses/bsd-license.php - -THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, -WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. - -*/ - -#ifndef BOOTGUARD_H -#define BOOTGUARD_H - -#include "basetypes.h" -#include "sha256.h" - -#pragma pack(push, 1) - -const UByteArray BG_VENDOR_HASH_FILE_GUID_PHOENIX // 389CC6F2-1EA8-467B-AB8A-78E769AE2A15 -("\xF2\xC6\x9C\x38\xA8\x1E\x7B\x46\xAB\x8A\x78\xE7\x69\xAE\x2A\x15", 16); - -#define BG_VENDOR_HASH_FILE_SIGNATURE_PHOENIX (*(UINT64 *)"$HASHTBL") - -const UByteArray BG_VENDOR_HASH_FILE_GUID_AMI // CBC91F44-A4BC-4A5B-8696-703451D0B053 -("\x44\x1F\xC9\xCB\xBC\xA4\x5B\x4A\x86\x96\x70\x34\x51\xD0\xB0\x53", 16); - -typedef struct BG_VENDOR_HASH_FILE_ENTRY -{ - UINT8 Hash[SHA256_DIGEST_SIZE]; - UINT32 Offset; - UINT32 Size; -} BG_VENDOR_HASH_FILE_ENTRY; - -typedef struct BG_VENDOR_HASH_FILE_HEADER_PHOENIX_ -{ - UINT64 Signature; - UINT32 NumEntries; - //BG_VENDOR_HASH_FILE_ENTRY Entries[]; -} BG_VENDOR_HASH_FILE_HEADER_PHOENIX; - -typedef struct BG_VENDOR_HASH_FILE_HEADER_AMI_NEW_ -{ - BG_VENDOR_HASH_FILE_ENTRY Entries[2]; -} BG_VENDOR_HASH_FILE_HEADER_AMI_NEW; - -typedef struct BG_VENDOR_HASH_FILE_HEADER_AMI_OLD_ -{ - UINT8 Hash[SHA256_DIGEST_SIZE]; - UINT32 Size; - // Offset is derived from flash map, will be detected as root volume with DXE core -} BG_VENDOR_HASH_FILE_HEADER_AMI_OLD; - -typedef struct BG_MICROSOFT_PMDA_HEADER_ -{ - UINT32 Version; - UINT32 NumEntries; -} BG_MICROSOFT_PMDA_HEADER; - -#define BG_MICROSOFT_PMDA_VERSION 0x00000001 - -typedef struct BG_MICROSOFT_PMDA_ENTRY_ -{ - UINT32 Address; - UINT32 Size; - UINT8 Hash[SHA256_DIGEST_SIZE]; -} BG_MICROSOFT_PMDA_ENTRY; - -// -// Intel ACM -// - -#define INTEL_ACM_MODULE_TYPE 0x2 -#define INTEL_ACM_MODULE_SUBTYPE_TXT_ACM 0x0 -#define INTEL_ACM_MODULE_SUBTYPE_S_ACM 0x1 -#define INTEL_ACM_MODULE_SUBTYPE_BOOTGUARD 0x3 -#define INTEL_ACM_MODULE_VENDOR 0x8086 - -typedef struct INTEL_ACM_HEADER_ { - UINT16 ModuleType; - UINT16 ModuleSubtype; - UINT32 HeaderType; - UINT32 HeaderVersion; - UINT16 ChipsetId; - UINT16 Flags; - UINT32 ModuleVendor; - UINT8 DateDay; - UINT8 DateMonth; - UINT16 DateYear; - UINT32 ModuleSize; - UINT16 AcmSvn; - UINT16 : 16; - UINT32 Unknown1; - UINT32 Unknown2; - UINT32 GdtMax; - UINT32 GdtBase; - UINT32 SegmentSel; - UINT32 EntryPoint; - UINT8 Unknown3[64]; - UINT32 KeySize; - UINT32 Unknown4; - UINT8 RsaPubKey[256]; - UINT32 RsaPubExp; - UINT8 RsaSig[256]; -} INTEL_ACM_HEADER; - -// -// Intel BootGuard Key Manifest -// -#define BG_BOOT_POLICY_MANIFEST_HEADER_TAG (*(UINT64 *)"__ACBP__") -typedef struct BG_BOOT_POLICY_MANIFEST_HEADER_ { - UINT64 Tag; - UINT8 Version; - UINT8 HeaderVersion; - UINT8 PMBPMVersion; - UINT8 BPSVN; - UINT8 ACMSVN; - UINT8 : 8; - UINT16 NEMDataSize; -} BG_BOOT_POLICY_MANIFEST_HEADER; - -typedef struct SHA256_HASH_ { - UINT16 HashAlgorithmId; - UINT16 Size; - UINT8 HashBuffer[32]; -} SHA256_HASH; - -typedef struct RSA_PUBLIC_KEY_ { - UINT8 Version; - UINT16 KeySize; - UINT32 Exponent; - UINT8 Modulus[256]; -} RSA_PUBLIC_KEY; - -typedef struct RSA_SIGNATURE_ { - UINT8 Version; - UINT16 KeySize; - UINT16 HashId; - UINT8 Signature[256]; -} RSA_SIGNATURE; - -typedef struct KEY_SIGNATURE_ { - UINT8 Version; - UINT16 KeyId; - RSA_PUBLIC_KEY PubKey; - UINT16 SigScheme; - RSA_SIGNATURE Signature; -} BG_KEY_SIGNATURE; - -#define BG_IBB_SEGMENT_FLAG_IBB 0x0 -#define BG_IBB_SEGMENT_FLAG_NON_IBB 0x1 -typedef struct BG_IBB_SEGMENT_ELEMENT_ { -UINT16: 16; - UINT16 Flags; - UINT32 Base; - UINT32 Size; -} BG_IBB_SEGMENT_ELEMENT; - -#define BG_BOOT_POLICY_MANIFEST_IBB_ELEMENT_TAG (*(UINT64 *)"__IBBS__") -#define BG_IBB_FLAG_AUTHORITY_MEASURE 0x4 - -typedef struct BG_IBB_ELEMENT_ { - UINT64 Tag; - UINT8 Version; - UINT16 : 16; - UINT8 Unknown; - UINT32 Flags; - UINT64 IbbMchBar; - UINT64 VtdBar; - UINT32 PmrlBase; - UINT32 PmrlLimit; - UINT64 Unknown3; - UINT64 Unknown4; - SHA256_HASH IbbHash; - UINT32 EntryPoint; - SHA256_HASH Digest; - UINT8 IbbSegCount; - // BG_IBB_SEGMENT_ELEMENT IbbSegment[]; -} BG_IBB_ELEMENT; - -#define BG_BOOT_POLICY_MANIFEST_PLATFORM_MANUFACTURER_ELEMENT_TAG (*(UINT64 *)"__PMDA__") -typedef struct BG_PLATFORM_MANUFACTURER_ELEMENT_ { - UINT64 Tag; - UINT8 Version; - UINT16 DataSize; -} BG_PLATFORM_MANUFACTURER_ELEMENT; - -#define BG_BOOT_POLICY_MANIFEST_SIGNATURE_ELEMENT_TAG (*(UINT64 *)"__PMSG__") -typedef struct BG_BOOT_POLICY_MANIFEST_SIGNATURE_ELEMENT_ { - UINT64 Tag; - UINT8 Version; - BG_KEY_SIGNATURE KeySignature; -} BG_BOOT_POLICY_MANIFEST_SIGNATURE_ELEMENT; - -#define BG_KEY_MANIFEST_TAG (*(UINT64 *)"__KEYM__") -typedef struct BG_KEY_MANIFEST_ { - UINT64 Tag; - UINT8 Version; - UINT8 KmVersion; - UINT8 KmSvn; - UINT8 KmId; - SHA256_HASH BpKeyHash; - BG_KEY_SIGNATURE KeyManifestSignature; -} BG_KEY_MANIFEST; - -#pragma pack(pop) - -#endif // BOOTGUARD_H \ No newline at end of file diff --git a/common/bstrlib/bstrlib.h b/common/bstrlib/bstrlib.h index 476cc4f..39fa835 100644 --- a/common/bstrlib/bstrlib.h +++ b/common/bstrlib/bstrlib.h @@ -22,7 +22,7 @@ extern "C" { #include #include #include -#include "../uinttypes.h" +#include "../basetypes.h" #if !defined (BSTRLIB_VSNP_OK) && !defined (BSTRLIB_NOVSNP) # if defined (__TURBOC__) && !defined (__BORLANDC__) diff --git a/common/bstrlib/bstrwrap.h b/common/bstrlib/bstrwrap.h index 3e74b58..7b99088 100644 --- a/common/bstrlib/bstrwrap.h +++ b/common/bstrlib/bstrwrap.h @@ -52,7 +52,7 @@ #include #include "bstrlib.h" #include "../ubytearray.h" -#include "../uinttypes.h" +#include "../basetypes.h" #ifdef __cplusplus @@ -365,6 +365,7 @@ struct CBString : public tagbstring { // QString compatibility methods const char *toLocal8Bit() const { return *this; } bool contains(const char *str) { return find(str) >= 0; } + bool startsWith(const char *str) { return find(str) == 0; } bool endsWith(const char *str) { int len = strlen(str); return (slen >= len && (find(str, slen - len) == (slen - len))); } bool isEmpty() const { return slen == 0; } void clear() { *this = ""; } diff --git a/common/descriptor.cpp b/common/descriptor.cpp index caa6fde..db452c2 100644 --- a/common/descriptor.cpp +++ b/common/descriptor.cpp @@ -246,5 +246,5 @@ UString jedecIdToUString(UINT8 vendorId, UINT8 deviceId0, UINT8 deviceId1) case 0x9D7019: return UString("ISSI Ix25WP256"); } - return UString("Unknown"); + return usprintf("Unknown %08Xh", jedecId); } diff --git a/common/digest/sha1.c b/common/digest/sha1.c new file mode 100644 index 0000000..683e2df --- /dev/null +++ b/common/digest/sha1.c @@ -0,0 +1,230 @@ +/* sha1.c + + Copyright (c) 2022, Nikolaj Schlej. All rights reserved. + This program and the accompanying materials + are licensed and made available under the terms and conditions of the BSD License + which accompanies this distribution. The full text of the license may be found at + http://opensource.org/licenses/bsd-license.php + + THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. + + */ + +// +// This implementations are based on LibTomCrypt that was released into +// public domain by Tom St Denis. +// +#include "sha1.h" +#include +#include + +/* ulong64: 64-bit data type */ +#ifdef _MSC_VER + #define CONST64(n) n ## ui64 + typedef unsigned __int64 ulong64; +#else + #define CONST64(n) n ## ULL + typedef uint64_t ulong64; +#endif + +typedef uint32_t ulong32; + +#define LOAD32H(x, y) \ + do { x = ((ulong32)((y)[0] & 255)<<24) | \ + ((ulong32)((y)[1] & 255)<<16) | \ + ((ulong32)((y)[2] & 255)<<8) | \ + ((ulong32)((y)[3] & 255)); } while(0) + +#ifndef MIN +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) +#endif + +#define ROL(x, y) ( (((ulong32)(x)<<(ulong32)((y)&31)) | (((ulong32)(x)&0xFFFFFFFFUL)>>(ulong32)((32-((y)&31))&31))) & 0xFFFFFFFFUL) +#define ROLc(x, y) ( (((ulong32)(x)<<(ulong32)((y)&31)) | (((ulong32)(x)&0xFFFFFFFFUL)>>(ulong32)((32-((y)&31))&31))) & 0xFFFFFFFFUL) + +#define STORE32H(x, y) \ + do { (y)[0] = (unsigned char)(((x)>>24)&255); (y)[1] = (unsigned char)(((x)>>16)&255); \ + (y)[2] = (unsigned char)(((x)>>8)&255); (y)[3] = (unsigned char)((x)&255); } while(0) + +#define STORE64H(x, y) \ +do { (y)[0] = (unsigned char)(((x)>>56)&255); (y)[1] = (unsigned char)(((x)>>48)&255); \ + (y)[2] = (unsigned char)(((x)>>40)&255); (y)[3] = (unsigned char)(((x)>>32)&255); \ + (y)[4] = (unsigned char)(((x)>>24)&255); (y)[5] = (unsigned char)(((x)>>16)&255); \ + (y)[6] = (unsigned char)(((x)>>8)&255); (y)[7] = (unsigned char)((x)&255); } while(0) + +#define F0(x,y,z) (z ^ (x & (y ^ z))) +#define F1(x,y,z) (x ^ y ^ z) +#define F2(x,y,z) ((x & y) | (z & (x | y))) +#define F3(x,y,z) (x ^ y ^ z) + +struct sha1_state { + ulong64 length; + ulong32 state[5], curlen; + unsigned char buf[64]; +}; + +static int s_sha1_compress(struct sha1_state *md, const unsigned char *buf) +{ + ulong32 a,b,c,d,e,W[80],i; + ulong32 t; + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD32H(W[i], buf + (4*i)); + } + + /* copy state */ + a = md->state[0]; + b = md->state[1]; + c = md->state[2]; + d = md->state[3]; + e = md->state[4]; + + /* expand it */ + for (i = 16; i < 80; i++) { + W[i] = ROL(W[i-3] ^ W[i-8] ^ W[i-14] ^ W[i-16], 1); + } + + /* compress */ + /* round one */ + #define FF0(a,b,c,d,e,i) e = (ROLc(a, 5) + F0(b,c,d) + e + W[i] + 0x5a827999UL); b = ROLc(b, 30); + #define FF1(a,b,c,d,e,i) e = (ROLc(a, 5) + F1(b,c,d) + e + W[i] + 0x6ed9eba1UL); b = ROLc(b, 30); + #define FF2(a,b,c,d,e,i) e = (ROLc(a, 5) + F2(b,c,d) + e + W[i] + 0x8f1bbcdcUL); b = ROLc(b, 30); + #define FF3(a,b,c,d,e,i) e = (ROLc(a, 5) + F3(b,c,d) + e + W[i] + 0xca62c1d6UL); b = ROLc(b, 30); + + for (i = 0; i < 20; ) { + FF0(a,b,c,d,e,i++); t = e; e = d; d = c; c = b; b = a; a = t; + } + for (; i < 40; ) { + FF1(a,b,c,d,e,i++); t = e; e = d; d = c; c = b; b = a; a = t; + } + for (; i < 60; ) { + FF2(a,b,c,d,e,i++); t = e; e = d; d = c; c = b; b = a; a = t; + } + for (; i < 80; ) { + FF3(a,b,c,d,e,i++); t = e; e = d; d = c; c = b; b = a; a = t; + } + + #undef FF0 + #undef FF1 + #undef FF2 + #undef FF3 + + /* store */ + md->state[0] = md->state[0] + a; + md->state[1] = md->state[1] + b; + md->state[2] = md->state[2] + c; + md->state[3] = md->state[3] + d; + md->state[4] = md->state[4] + e; + + return 0; +} + +static int sha1_init(struct sha1_state * md) +{ + if (md == NULL) return -1; + md->state[0] = 0x67452301UL; + md->state[1] = 0xefcdab89UL; + md->state[2] = 0x98badcfeUL; + md->state[3] = 0x10325476UL; + md->state[4] = 0xc3d2e1f0UL; + md->curlen = 0; + md->length = 0; + return 0; +} + +static int sha1_process(struct sha1_state * md, const unsigned char *in, unsigned long inlen) +{ + unsigned long n; + int err; + if (md == NULL) return -1; + if (in == NULL) return -1; + if (md->curlen > sizeof(md->buf)) { + return -1; + } + if (((md->length + inlen * 8) < md->length) + || ((inlen * 8) < inlen)) { + return -1; + } + while (inlen > 0) { + if (md->curlen == 0 && inlen >= 64) { + if ((err = s_sha1_compress(md, in)) != 0) { + return err; + } + md->length += 64 * 8; + in += 64; + inlen -= 64; + } else { + n = MIN(inlen, (64 - md->curlen)); + memcpy(md->buf + md->curlen, in, (size_t)n); + md->curlen += n; + in += n; + inlen -= n; + if (md->curlen == 64) { + if ((err = s_sha1_compress(md, md->buf)) != 0) { + return err; + } + md->length += 8 * 64; + md->curlen = 0; + } + } + } + return 0; +} + +static int sha1_done(struct sha1_state * md, unsigned char *out) +{ + int i; + + if (md == NULL) return -1; + if (out == NULL) return -1; + + if (md->curlen >= sizeof(md->buf)) { + return -1; + } + + /* increase the length of the message */ + md->length += md->curlen * 8; + + /* append the '1' bit */ + md->buf[md->curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->curlen > 56) { + while (md->curlen < 64) { + md->buf[md->curlen++] = (unsigned char)0; + } + s_sha1_compress(md, md->buf); + md->curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->curlen < 56) { + md->buf[md->curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->length, md->buf+56); + s_sha1_compress(md, md->buf); + + /* copy output */ + for (i = 0; i < 5; i++) { + STORE32H(md->state[i], out+(4*i)); + } + + return 0; +} + +void sha1(const void *in, unsigned long inlen, void* out) +{ + struct sha1_state ctx; + sha1_init(&ctx); + sha1_process(&ctx, (const unsigned char*)in, inlen); + sha1_done(&ctx, (unsigned char *)out); +} + + diff --git a/common/uinttypes.h b/common/digest/sha1.h similarity index 50% rename from common/uinttypes.h rename to common/digest/sha1.h index 10b0a32..e698090 100644 --- a/common/uinttypes.h +++ b/common/digest/sha1.h @@ -1,6 +1,6 @@ -/* uinttypes.h +/* sha1.h -Copyright (c) 2021, Nikolaj Schlej. All rights reserved. +Copyright (c) 2022, Nikolaj Schlej. All rights reserved. This program and the accompanying materials are licensed and made available under the terms and conditions of the BSD License which accompanies this distribution. The full text of the license may be found at @@ -8,20 +8,18 @@ http://opensource.org/licenses/bsd-license.php THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. + */ -#ifndef UINTTYPES_H -#define UINTTYPES_H - -// A workaround for compilers not supporting c++11 and c11 -// for using PRIX64. -#define __STDC_FORMAT_MACROS -#include - -#if defined(__clang__) || defined(__GNUC__) -#define ATTRIBUTE_FORMAT_(t,f,a) __attribute__((format(t, f, a))) -#else -#define ATTRIBUTE_FORMAT_(t,f,a) +#ifndef SHA1_H +#define SHA1_H +#ifdef __cplusplus +extern "C" { #endif -#endif // UINTTYPES_H +void sha1(const void *in, unsigned long inlen, void* out); + +#ifdef __cplusplus +} +#endif +#endif // SHA2_H diff --git a/common/sha256.h b/common/digest/sha2.h similarity index 69% rename from common/sha256.h rename to common/digest/sha2.h index 7ebe423..59b4c5d 100644 --- a/common/sha256.h +++ b/common/digest/sha2.h @@ -1,6 +1,6 @@ -/* sha256.h +/* sha2.h -Copyright (c) 2017, LongSoft. All rights reserved. +Copyright (c) 2017, Nikolaj Schlej. All rights reserved. This program and the accompanying materials are licensed and made available under the terms and conditions of the BSD License which accompanies this distribution. The full text of the license may be found at @@ -11,17 +11,17 @@ WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. */ -#ifndef SHA256_H -#define SHA256_H +#ifndef SHA2_H +#define SHA2_H #ifdef __cplusplus extern "C" { #endif -#define SHA256_DIGEST_SIZE 32 - void sha256(const void *in, unsigned long inlen, void* out); - +void sha384(const void *in, unsigned long inlen, void* out); +void sha512(const void *in, unsigned long inlen, void* out); + #ifdef __cplusplus } #endif -#endif // SHA256_H +#endif // SHA2_H diff --git a/common/digest/sha256.c b/common/digest/sha256.c new file mode 100644 index 0000000..a9b086d --- /dev/null +++ b/common/digest/sha256.c @@ -0,0 +1,279 @@ +/* sha256.c + + Copyright (c) 2017, Nikolaj Schlej. All rights reserved. + This program and the accompanying materials + are licensed and made available under the terms and conditions of the BSD License + which accompanies this distribution. The full text of the license may be found at + http://opensource.org/licenses/bsd-license.php + + THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. + + */ + +// +// This implementations are based on LibTomCrypt that was released into +// public domain by Tom St Denis. +// + +#include "sha2.h" +#include +#include + +/* ulong64: 64-bit data type */ +#ifdef _MSC_VER + #define CONST64(n) n ## ui64 + typedef unsigned __int64 ulong64; +#else + #define CONST64(n) n ## ULL + typedef uint64_t ulong64; +#endif + +typedef uint32_t ulong32; + +#define LOAD32H(x, y) \ + do { x = ((ulong32)((y)[0] & 255)<<24) | \ + ((ulong32)((y)[1] & 255)<<16) | \ + ((ulong32)((y)[2] & 255)<<8) | \ + ((ulong32)((y)[3] & 255)); } while(0) + +#ifndef MIN +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) +#endif + +#define RORc(x, y) ( ((((ulong32)(x)&0xFFFFFFFFUL)>>(ulong32)((y)&31)) | ((ulong32)(x)<<(ulong32)((32-((y)&31))&31))) & 0xFFFFFFFFUL) + +#define STORE32H(x, y) \ + do { (y)[0] = (unsigned char)(((x)>>24)&255); (y)[1] = (unsigned char)(((x)>>16)&255); \ + (y)[2] = (unsigned char)(((x)>>8)&255); (y)[3] = (unsigned char)((x)&255); } while(0) + +#define STORE64H(x, y) \ +do { (y)[0] = (unsigned char)(((x)>>56)&255); (y)[1] = (unsigned char)(((x)>>48)&255); \ + (y)[2] = (unsigned char)(((x)>>40)&255); (y)[3] = (unsigned char)(((x)>>32)&255); \ + (y)[4] = (unsigned char)(((x)>>24)&255); (y)[5] = (unsigned char)(((x)>>16)&255); \ + (y)[6] = (unsigned char)(((x)>>8)&255); (y)[7] = (unsigned char)((x)&255); } while(0) + +/* Various logical functions */ +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) RORc((x),(n)) +#define R(x, n) (((x)&0xFFFFFFFFUL)>>(n)) +#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) +#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) + +struct sha256_state { + ulong64 length; + ulong32 state[8], curlen; + unsigned char buf[32*2]; +}; + +/* compress 512-bits */ +static int s_sha256_compress(struct sha256_state * md, const unsigned char *buf) +{ + ulong32 S[8], W[64], t0, t1; + int i; + + /* copy state into S */ + for (i = 0; i < 8; i++) { + S[i] = md->state[i]; + } + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD32H(W[i], buf + (4*i)); + } + + /* fill W[16..63] */ + for (i = 16; i < 64; i++) { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; + } + + /* Compress */ +#define RND(a,b,c,d,e,f,g,h,i,ki) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],0,0x428a2f98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],1,0x71374491); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],2,0xb5c0fbcf); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],3,0xe9b5dba5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],4,0x3956c25b); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],5,0x59f111f1); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],6,0x923f82a4); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],7,0xab1c5ed5); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],8,0xd807aa98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],9,0x12835b01); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],10,0x243185be); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],11,0x550c7dc3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],12,0x72be5d74); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],13,0x80deb1fe); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],14,0x9bdc06a7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],15,0xc19bf174); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],16,0xe49b69c1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],17,0xefbe4786); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],18,0x0fc19dc6); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],19,0x240ca1cc); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],20,0x2de92c6f); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],21,0x4a7484aa); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],22,0x5cb0a9dc); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],23,0x76f988da); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],24,0x983e5152); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],25,0xa831c66d); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],26,0xb00327c8); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],27,0xbf597fc7); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],28,0xc6e00bf3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],29,0xd5a79147); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],30,0x06ca6351); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],31,0x14292967); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],32,0x27b70a85); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],33,0x2e1b2138); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],34,0x4d2c6dfc); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],35,0x53380d13); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],36,0x650a7354); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],37,0x766a0abb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],38,0x81c2c92e); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],39,0x92722c85); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],40,0xa2bfe8a1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],41,0xa81a664b); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],42,0xc24b8b70); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],43,0xc76c51a3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],44,0xd192e819); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],45,0xd6990624); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],46,0xf40e3585); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],47,0x106aa070); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],48,0x19a4c116); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],49,0x1e376c08); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],50,0x2748774c); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],51,0x34b0bcb5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],52,0x391c0cb3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],53,0x4ed8aa4a); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],54,0x5b9cca4f); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],55,0x682e6ff3); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],56,0x748f82ee); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],57,0x78a5636f); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],58,0x84c87814); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],59,0x8cc70208); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],60,0x90befffa); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],61,0xa4506ceb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],62,0xbef9a3f7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],63,0xc67178f2); +#undef RND + + /* feedback */ + for (i = 0; i < 8; i++) { + md->state[i] = md->state[i] + S[i]; + } + return 0; +} + +static int sha256_init(struct sha256_state * md) +{ + if (md == NULL) return -1; + md->curlen = 0; + md->length = 0; + md->state[0] = 0x6A09E667UL; + md->state[1] = 0xBB67AE85UL; + md->state[2] = 0x3C6EF372UL; + md->state[3] = 0xA54FF53AUL; + md->state[4] = 0x510E527FUL; + md->state[5] = 0x9B05688CUL; + md->state[6] = 0x1F83D9ABUL; + md->state[7] = 0x5BE0CD19UL; + return 0; +} + +static int sha256_process(struct sha256_state * md, const unsigned char *in, unsigned long inlen) +{ + unsigned long n; + int err; + if (md == NULL) return -1; + if (in == NULL) return -1; + if (md->curlen > sizeof(md->buf)) { + return -1; + } + if (((md->length + inlen * 8) < md->length) + || ((inlen * 8) < inlen)) { + return -1; + } + while (inlen > 0) { + if (md->curlen == 0 && inlen >= 64) { + if ((err = s_sha256_compress(md, in)) != 0) { + return err; + } + md->length += 64 * 8; + in += 64; + inlen -= 64; + } else { + n = MIN(inlen, (64 - md->curlen)); + memcpy(md->buf + md->curlen, in, (size_t)n); + md->curlen += n; + in += n; + inlen -= n; + if (md->curlen == 64) { + if ((err = s_sha256_compress(md, md->buf)) != 0) { + return err; + } + md->length += 8 * 64; + md->curlen = 0; + } + } + } + return 0; +} + +static int sha256_done(struct sha256_state * md, unsigned char *out) +{ + int i; + + if (md == NULL) return -1; + if (out == NULL) return -1; + + if (md->curlen >= sizeof(md->buf)) { + return -1; + } + + /* increase the length of the message */ + md->length += md->curlen * 8; + + /* append the '1' bit */ + md->buf[md->curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->curlen > 56) { + while (md->curlen < 64) { + md->buf[md->curlen++] = (unsigned char)0; + } + s_sha256_compress(md, md->buf); + md->curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->curlen < 56) { + md->buf[md->curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->length, md->buf+56); + s_sha256_compress(md, md->buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE32H(md->state[i], out+(4*i)); + } + return 0; +} + +void sha256(const void *in, unsigned long inlen, void* out) +{ + struct sha256_state ctx; + sha256_init(&ctx); + sha256_process(&ctx, (const unsigned char*)in, inlen); + sha256_done(&ctx, (unsigned char *)out); +} diff --git a/common/digest/sha512.c b/common/digest/sha512.c new file mode 100644 index 0000000..e99ea14 --- /dev/null +++ b/common/digest/sha512.c @@ -0,0 +1,312 @@ +/* sha512.c + + Copyright (c) 2022, Nikolaj Schlej. All rights reserved. + This program and the accompanying materials + are licensed and made available under the terms and conditions of the BSD License + which accompanies this distribution. The full text of the license may be found at + http://opensource.org/licenses/bsd-license.php + + THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. + + */ + +// +// This implementations are based on LibTomCrypt that was released into +// public domain by Tom St Denis. +// + +#include "sha2.h" +#include +#include + +/* ulong64: 64-bit data type */ +#ifdef _MSC_VER + #define CONST64(n) n ## ui64 + typedef unsigned __int64 ulong64; + typedef __int64 long64; +#else + #define CONST64(n) n ## ULL + typedef unsigned long long ulong64; + typedef long long long64; +#endif + +#define ROR64c(x, y) \ + ( ((((x)&CONST64(0xFFFFFFFFFFFFFFFF))>>((ulong64)(y)&CONST64(63))) | \ + ((x)<<(((ulong64)64-((y)&63))&63))) & CONST64(0xFFFFFFFFFFFFFFFF)) + +#define LOAD64H(x, y) \ +do { x = (((ulong64)((y)[0] & 255))<<56)|(((ulong64)((y)[1] & 255))<<48) | \ + (((ulong64)((y)[2] & 255))<<40)|(((ulong64)((y)[3] & 255))<<32) | \ + (((ulong64)((y)[4] & 255))<<24)|(((ulong64)((y)[5] & 255))<<16) | \ + (((ulong64)((y)[6] & 255))<<8)|(((ulong64)((y)[7] & 255))); } while(0) + +#define STORE64H(x, y) \ +do { (y)[0] = (unsigned char)(((x)>>56)&255); (y)[1] = (unsigned char)(((x)>>48)&255); \ + (y)[2] = (unsigned char)(((x)>>40)&255); (y)[3] = (unsigned char)(((x)>>32)&255); \ + (y)[4] = (unsigned char)(((x)>>24)&255); (y)[5] = (unsigned char)(((x)>>16)&255); \ + (y)[6] = (unsigned char)(((x)>>8)&255); (y)[7] = (unsigned char)((x)&255); } while(0) + +#ifndef MIN +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) +#endif + +/* the K array */ +static const ulong64 K[80] = { +CONST64(0x428a2f98d728ae22), CONST64(0x7137449123ef65cd), +CONST64(0xb5c0fbcfec4d3b2f), CONST64(0xe9b5dba58189dbbc), +CONST64(0x3956c25bf348b538), CONST64(0x59f111f1b605d019), +CONST64(0x923f82a4af194f9b), CONST64(0xab1c5ed5da6d8118), +CONST64(0xd807aa98a3030242), CONST64(0x12835b0145706fbe), +CONST64(0x243185be4ee4b28c), CONST64(0x550c7dc3d5ffb4e2), +CONST64(0x72be5d74f27b896f), CONST64(0x80deb1fe3b1696b1), +CONST64(0x9bdc06a725c71235), CONST64(0xc19bf174cf692694), +CONST64(0xe49b69c19ef14ad2), CONST64(0xefbe4786384f25e3), +CONST64(0x0fc19dc68b8cd5b5), CONST64(0x240ca1cc77ac9c65), +CONST64(0x2de92c6f592b0275), CONST64(0x4a7484aa6ea6e483), +CONST64(0x5cb0a9dcbd41fbd4), CONST64(0x76f988da831153b5), +CONST64(0x983e5152ee66dfab), CONST64(0xa831c66d2db43210), +CONST64(0xb00327c898fb213f), CONST64(0xbf597fc7beef0ee4), +CONST64(0xc6e00bf33da88fc2), CONST64(0xd5a79147930aa725), +CONST64(0x06ca6351e003826f), CONST64(0x142929670a0e6e70), +CONST64(0x27b70a8546d22ffc), CONST64(0x2e1b21385c26c926), +CONST64(0x4d2c6dfc5ac42aed), CONST64(0x53380d139d95b3df), +CONST64(0x650a73548baf63de), CONST64(0x766a0abb3c77b2a8), +CONST64(0x81c2c92e47edaee6), CONST64(0x92722c851482353b), +CONST64(0xa2bfe8a14cf10364), CONST64(0xa81a664bbc423001), +CONST64(0xc24b8b70d0f89791), CONST64(0xc76c51a30654be30), +CONST64(0xd192e819d6ef5218), CONST64(0xd69906245565a910), +CONST64(0xf40e35855771202a), CONST64(0x106aa07032bbd1b8), +CONST64(0x19a4c116b8d2d0c8), CONST64(0x1e376c085141ab53), +CONST64(0x2748774cdf8eeb99), CONST64(0x34b0bcb5e19b48a8), +CONST64(0x391c0cb3c5c95a63), CONST64(0x4ed8aa4ae3418acb), +CONST64(0x5b9cca4f7763e373), CONST64(0x682e6ff3d6b2b8a3), +CONST64(0x748f82ee5defb2fc), CONST64(0x78a5636f43172f60), +CONST64(0x84c87814a1f0ab72), CONST64(0x8cc702081a6439ec), +CONST64(0x90befffa23631e28), CONST64(0xa4506cebde82bde9), +CONST64(0xbef9a3f7b2c67915), CONST64(0xc67178f2e372532b), +CONST64(0xca273eceea26619c), CONST64(0xd186b8c721c0c207), +CONST64(0xeada7dd6cde0eb1e), CONST64(0xf57d4f7fee6ed178), +CONST64(0x06f067aa72176fba), CONST64(0x0a637dc5a2c898a6), +CONST64(0x113f9804bef90dae), CONST64(0x1b710b35131c471b), +CONST64(0x28db77f523047d84), CONST64(0x32caab7b40c72493), +CONST64(0x3c9ebe0a15c9bebc), CONST64(0x431d67c49c100d4c), +CONST64(0x4cc5d4becb3e42b6), CONST64(0x597f299cfc657e2a), +CONST64(0x5fcb6fab3ad6faec), CONST64(0x6c44198c4a475817) +}; + +/* Various logical functions */ +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) ROR64c(x, n) +#define R(x, n) (((x)&CONST64(0xFFFFFFFFFFFFFFFF))>>((ulong64)n)) +#define Sigma0(x) (S(x, 28) ^ S(x, 34) ^ S(x, 39)) +#define Sigma1(x) (S(x, 14) ^ S(x, 18) ^ S(x, 41)) +#define Gamma0(x) (S(x, 1) ^ S(x, 8) ^ R(x, 7)) +#define Gamma1(x) (S(x, 19) ^ S(x, 61) ^ R(x, 6)) + +struct sha512_state { + ulong64 length, state[8]; + unsigned long curlen; + unsigned char buf[128]; +}; + +/* compress 1024-bits */ +static int s_sha512_compress(struct sha512_state * md, const unsigned char *buf) +{ + ulong64 S[8], W[80], t0, t1; + int i; + + /* copy state into S */ + for (i = 0; i < 8; i++) { + S[i] = md->state[i]; + } + + /* copy the state into 1024-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD64H(W[i], buf + (8*i)); + } + + /* fill W[16..79] */ + for (i = 16; i < 80; i++) { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; + } + + /* Compress */ +#define RND(a,b,c,d,e,f,g,h,i) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + for (i = 0; i < 80; i += 8) { + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],i+0); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],i+1); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],i+2); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],i+3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],i+4); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],i+5); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],i+6); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],i+7); + } + + /* feedback */ + for (i = 0; i < 8; i++) { + md->state[i] = md->state[i] + S[i]; + } + + return 0; +} + +static int sha512_init(struct sha512_state * md) +{ + if (md == NULL) return -1; + md->curlen = 0; + md->length = 0; + md->state[0] = CONST64(0x6a09e667f3bcc908); + md->state[1] = CONST64(0xbb67ae8584caa73b); + md->state[2] = CONST64(0x3c6ef372fe94f82b); + md->state[3] = CONST64(0xa54ff53a5f1d36f1); + md->state[4] = CONST64(0x510e527fade682d1); + md->state[5] = CONST64(0x9b05688c2b3e6c1f); + md->state[6] = CONST64(0x1f83d9abfb41bd6b); + md->state[7] = CONST64(0x5be0cd19137e2179); + return 0; +} + +static int sha512_process(struct sha512_state * md, const unsigned char *in, unsigned long inlen) +{ + unsigned long n; + int err; + if (md == NULL) return -1; + if (in == NULL) return -1; + if (md->curlen > sizeof(md->buf)) { + return -1; + } + if (((md->length + inlen * 8) < md->length) + || ((inlen * 8) < inlen)) { + return -1; + } + while (inlen > 0) { + if (md->curlen == 0 && inlen >= 128) { + if ((err = s_sha512_compress(md, in)) != 0) { + return err; + } + md->length += 128 * 8; + in += 128; + inlen -= 128; + } else { + n = MIN(inlen, (128 - md->curlen)); + memcpy(md->buf + md->curlen, in, (size_t)n); + md->curlen += n; + in += n; + inlen -= n; + if (md->curlen == 128) { + if ((err = s_sha512_compress(md, md->buf)) != 0) { + return err; + } + md->length += 8 * 128; + md->curlen = 0; + } + } + } + return 0; +} + +static int sha512_done(struct sha512_state * md, unsigned char *out) +{ + int i; + + if (md == NULL) return -1; + if (out == NULL) return -1; + + if (md->curlen >= sizeof(md->buf)) { + return -1; + } + + /* increase the length of the message */ + md->length += md->curlen * CONST64(8); + + /* append the '1' bit */ + md->buf[md->curlen++] = (unsigned char)0x80; + + /* if the length is currently above 112 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->curlen > 112) { + while (md->curlen < 128) { + md->buf[md->curlen++] = (unsigned char)0; + } + s_sha512_compress(md, md->buf); + md->curlen = 0; + } + + /* pad upto 120 bytes of zeroes + * note: that from 112 to 120 is the 64 MSB of the length. We assume that you won't hash + * > 2^64 bits of data... :-) + */ + while (md->curlen < 120) { + md->buf[md->curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->length, md->buf+120); + s_sha512_compress(md, md->buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE64H(md->state[i], out+(8*i)); + } + + return 0; +} + +static int sha384_init(struct sha512_state * md) +{ + if (md == NULL) return -1; + + md->curlen = 0; + md->length = 0; + md->state[0] = CONST64(0xcbbb9d5dc1059ed8); + md->state[1] = CONST64(0x629a292a367cd507); + md->state[2] = CONST64(0x9159015a3070dd17); + md->state[3] = CONST64(0x152fecd8f70e5939); + md->state[4] = CONST64(0x67332667ffc00b31); + md->state[5] = CONST64(0x8eb44a8768581511); + md->state[6] = CONST64(0xdb0c2e0d64f98fa7); + md->state[7] = CONST64(0x47b5481dbefa4fa4); + return 0; +} + +static int sha384_done(struct sha512_state * md, unsigned char *out) +{ + unsigned char buf[64]; + + if (md == NULL) return -1; + if (out == NULL) return -1;; + + if (md->curlen >= sizeof(md->buf)) { + return -1; + } + + sha512_done(md, buf); + memcpy(out, buf, 48); + return 0; +} + +void sha384(const void *in, unsigned long inlen, void* out) +{ + struct sha512_state ctx; + sha384_init(&ctx); + sha512_process(&ctx, (const unsigned char*)in, inlen); + sha384_done(&ctx, (unsigned char *)out); +} + +void sha512(const void *in, unsigned long inlen, void* out) +{ + struct sha512_state ctx; + sha512_init(&ctx); + sha512_process(&ctx, (const unsigned char*)in, inlen); + sha512_done(&ctx, (unsigned char *)out); +} diff --git a/common/digest/sm3.c b/common/digest/sm3.c new file mode 100644 index 0000000..9f25e6b --- /dev/null +++ b/common/digest/sm3.c @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd + */ +#include "sm3.h" +#include + +struct sm3_context { + uint32_t total[2]; /* number of bytes processed */ + uint32_t state[8]; /* intermediate digest state */ + uint8_t buffer[64]; /* data block being processed */ + uint8_t ipad[64]; /* HMAC: inner padding */ + uint8_t opad[64]; /* HMAC: outer padding */ +}; + +static void sm3_init(struct sm3_context *ctx); +static void sm3_update(struct sm3_context *ctx, const uint8_t *input, size_t ilen); +static void sm3_final(struct sm3_context *ctx, uint8_t* output); + +#define GET_UINT32_BE(n, b, i) \ + do { \ + (n) = ((uint32_t)(b)[(i)] << 24) | \ + ((uint32_t)(b)[(i) + 1] << 16) | \ + ((uint32_t)(b)[(i) + 2] << 8) | \ + ((uint32_t)(b)[(i) + 3]); \ + } while (0) + +#define PUT_UINT32_BE(n, b, i) \ + do { \ + (b)[(i)] = (uint8_t)((n) >> 24); \ + (b)[(i) + 1] = (uint8_t)((n) >> 16); \ + (b)[(i) + 2] = (uint8_t)((n) >> 8); \ + (b)[(i) + 3] = (uint8_t)((n)); \ + } while (0) + +static void sm3_init(struct sm3_context *ctx) +{ + ctx->total[0] = 0; + ctx->total[1] = 0; + + ctx->state[0] = 0x7380166F; + ctx->state[1] = 0x4914B2B9; + ctx->state[2] = 0x172442D7; + ctx->state[3] = 0xDA8A0600; + ctx->state[4] = 0xA96F30BC; + ctx->state[5] = 0x163138AA; + ctx->state[6] = 0xE38DEE4D; + ctx->state[7] = 0xB0FB0E4E; +} + +static void sm3_process(struct sm3_context *ctx, const uint8_t data[64]) +{ + uint32_t SS1, SS2, TT1, TT2, W[68], W1[64]; + uint32_t A, B, C, D, E, F, G, H; + uint32_t T[64]; + uint32_t Temp1, Temp2, Temp3, Temp4, Temp5; + int j; + + for (j = 0; j < 16; j++) + T[j] = 0x79CC4519; + for (j = 16; j < 64; j++) + T[j] = 0x7A879D8A; + + GET_UINT32_BE(W[0], data, 0); + GET_UINT32_BE(W[1], data, 4); + GET_UINT32_BE(W[2], data, 8); + GET_UINT32_BE(W[3], data, 12); + GET_UINT32_BE(W[4], data, 16); + GET_UINT32_BE(W[5], data, 20); + GET_UINT32_BE(W[6], data, 24); + GET_UINT32_BE(W[7], data, 28); + GET_UINT32_BE(W[8], data, 32); + GET_UINT32_BE(W[9], data, 36); + GET_UINT32_BE(W[10], data, 40); + GET_UINT32_BE(W[11], data, 44); + GET_UINT32_BE(W[12], data, 48); + GET_UINT32_BE(W[13], data, 52); + GET_UINT32_BE(W[14], data, 56); + GET_UINT32_BE(W[15], data, 60); + +#define FF0(x, y, z) ((x) ^ (y) ^ (z)) +#define FF1(x, y, z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) + +#define GG0(x, y, z) ((x) ^ (y) ^ (z)) +#define GG1(x, y, z) (((x) & (y)) | ((~(x)) & (z))) + +#define SHL(x, n) ((x) << (n)) +#define ROTL(x, y) ( (((uint32_t)(x)<<(uint32_t)((y)&31)) | (((uint32_t)(x)&0xFFFFFFFFUL)>>(uint32_t)((32-((y)&31))&31))) & 0xFFFFFFFFUL) + +#define P0(x) ((x) ^ ROTL((x), 9) ^ ROTL((x), 17)) +#define P1(x) ((x) ^ ROTL((x), 15) ^ ROTL((x), 23)) + + for (j = 16; j < 68; j++) { + /* + * W[j] = P1( W[j-16] ^ W[j-9] ^ ROTL(W[j-3],15)) ^ + * ROTL(W[j - 13],7 ) ^ W[j-6]; + */ + + Temp1 = W[j - 16] ^ W[j - 9]; + Temp2 = ROTL(W[j - 3], 15); + Temp3 = Temp1 ^ Temp2; + Temp4 = P1(Temp3); + Temp5 = ROTL(W[j - 13], 7) ^ W[j - 6]; + W[j] = Temp4 ^ Temp5; + } + + for (j = 0; j < 64; j++) + W1[j] = W[j] ^ W[j + 4]; + + A = ctx->state[0]; + B = ctx->state[1]; + C = ctx->state[2]; + D = ctx->state[3]; + E = ctx->state[4]; + F = ctx->state[5]; + G = ctx->state[6]; + H = ctx->state[7]; + + for (j = 0; j < 16; j++) { + SS1 = ROTL(ROTL(A, 12) + E + ROTL(T[j], j), 7); + SS2 = SS1 ^ ROTL(A, 12); + TT1 = FF0(A, B, C) + D + SS2 + W1[j]; + TT2 = GG0(E, F, G) + H + SS1 + W[j]; + D = C; + C = ROTL(B, 9); + B = A; + A = TT1; + H = G; + G = ROTL(F, 19); + F = E; + E = P0(TT2); + } + + for (j = 16; j < 64; j++) { + SS1 = ROTL(ROTL(A, 12) + E + ROTL(T[j], j), 7); + SS2 = SS1 ^ ROTL(A, 12); + TT1 = FF1(A, B, C) + D + SS2 + W1[j]; + TT2 = GG1(E, F, G) + H + SS1 + W[j]; + D = C; + C = ROTL(B, 9); + B = A; + A = TT1; + H = G; + G = ROTL(F, 19); + F = E; + E = P0(TT2); + } + + ctx->state[0] ^= A; + ctx->state[1] ^= B; + ctx->state[2] ^= C; + ctx->state[3] ^= D; + ctx->state[4] ^= E; + ctx->state[5] ^= F; + ctx->state[6] ^= G; + ctx->state[7] ^= H; +} + +static void sm3_update(struct sm3_context *ctx, const uint8_t *input, size_t ilen) +{ + size_t fill; + size_t left; + + if (!ilen) + return; + + left = ctx->total[0] & 0x3F; + fill = 64 - left; + + ctx->total[0] += ilen; + + if (ctx->total[0] < ilen) + ctx->total[1]++; + + if (left && ilen >= fill) { + memcpy(ctx->buffer + left, input, fill); + sm3_process(ctx, ctx->buffer); + input += fill; + ilen -= fill; + left = 0; + } + + while (ilen >= 64) { + sm3_process(ctx, input); + input += 64; + ilen -= 64; + } + + if (ilen > 0) + memcpy(ctx->buffer + left, input, ilen); +} + +static const uint8_t sm3_padding[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +static void sm3_final(struct sm3_context *ctx, uint8_t* output) +{ + uint32_t last, padn; + uint32_t high, low; + uint8_t msglen[8]; + + high = (ctx->total[0] >> 29) | (ctx->total[1] << 3); + low = ctx->total[0] << 3; + + PUT_UINT32_BE(high, msglen, 0); + PUT_UINT32_BE(low, msglen, 4); + + last = ctx->total[0] & 0x3F; + padn = (last < 56) ? (56 - last) : (120 - last); + + sm3_update(ctx, sm3_padding, padn); + sm3_update(ctx, msglen, 8); + + PUT_UINT32_BE(ctx->state[0], output, 0); + PUT_UINT32_BE(ctx->state[1], output, 4); + PUT_UINT32_BE(ctx->state[2], output, 8); + PUT_UINT32_BE(ctx->state[3], output, 12); + PUT_UINT32_BE(ctx->state[4], output, 16); + PUT_UINT32_BE(ctx->state[5], output, 20); + PUT_UINT32_BE(ctx->state[6], output, 24); + PUT_UINT32_BE(ctx->state[7], output, 28); +} + +void sm3(const void *in, unsigned long inlen, void* out) +{ + struct sm3_context ctx; + sm3_init(&ctx); + sm3_update(&ctx, (const uint8_t *)in, (size_t)inlen); + sm3_final(&ctx, (uint8_t*)out); +} diff --git a/common/digest/sm3.h b/common/digest/sm3.h new file mode 100644 index 0000000..0d248b6 --- /dev/null +++ b/common/digest/sm3.h @@ -0,0 +1,32 @@ +/* sm3.h + +Copyright (c) 2022, Nikolaj Schlej. All rights reserved. +This program and the accompanying materials +are licensed and made available under the terms and conditions of the BSD License +which accompanies this distribution. The full text of the license may be found at +http://opensource.org/licenses/bsd-license.php + +THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, +WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. + +*/ + +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd + */ +#ifndef SM3_H +#define SM3_H +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +void sm3(const void *in, unsigned long inlen, void* out); + +#ifdef __cplusplus +} +#endif +#endif /* SM3_H */ diff --git a/common/ffs.cpp b/common/ffs.cpp index a2b5a8c..2425182 100644 --- a/common/ffs.cpp +++ b/common/ffs.cpp @@ -14,9 +14,100 @@ #include "ffs.h" #include "guiddatabase.h" +#include "ubytearray.h" -// This is a workaround for the lack of static std::vector initializer before C++11 -const UByteArray FFSv2VolumesInt[] = { +// +// GUIDs mentioned in by ffs.h +// +// Standard FMP capsule GUID +extern const UByteArray EFI_FMP_CAPSULE_GUID // 6DCBD5ED-E82D-4C44-BDA1-7194199AD92A +("\xED\xD5\xCB\x6D\x2D\xE8\x44\x4C\xBD\xA1\x71\x94\x19\x9A\xD9\x2A", 16); +// Standard EFI capsule GUID +extern const UByteArray EFI_CAPSULE_GUID // 3B6686BD-0D76-4030-B70E-B5519E2FC5A0 +("\xBD\x86\x66\x3B\x76\x0D\x30\x40\xB7\x0E\xB5\x51\x9E\x2F\xC5\xA0", 16); +// Intel capsule GUID +extern const UByteArray INTEL_CAPSULE_GUID // 539182B9-ABB5-4391-B69A-E3A943F72FCC +("\xB9\x82\x91\x53\xB5\xAB\x91\x43\xB6\x9A\xE3\xA9\x43\xF7\x2F\xCC", 16); +// Lenovo capsule GUID +extern const UByteArray LENOVO_CAPSULE_GUID // E20BAFD3-9914-4F4F-9537-3129E090EB3C +("\xD3\xAF\x0B\xE2\x14\x99\x4F\x4F\x95\x37\x31\x29\xE0\x90\xEB\x3C", 16); +// Another Lenovo capsule GUID +extern const UByteArray LENOVO2_CAPSULE_GUID // 25B5FE76-8243-4A5C-A9BD-7EE3246198B5 +("\x76\xFE\xB5\x25\x43\x82\x5C\x4A\xA9\xBD\x7E\xE3\x24\x61\x98\xB5", 16); +// Toshiba capsule GUID +extern const UByteArray TOSHIBA_CAPSULE_GUID // 3BE07062-1D51-45D2-832B-F093257ED461 +("\x62\x70\xE0\x3B\x51\x1D\xD2\x45\x83\x2B\xF0\x93\x25\x7E\xD4\x61", 16); +// AMI Aptio signed extended capsule GUID +extern const UByteArray APTIO_SIGNED_CAPSULE_GUID // 4A3CA68B-7723-48FB-803D-578CC1FEC44D +("\x8B\xA6\x3C\x4A\x23\x77\xFB\x48\x80\x3D\x57\x8C\xC1\xFE\xC4\x4D", 16); +// AMI Aptio unsigned extended capsule GUID +extern const UByteArray APTIO_UNSIGNED_CAPSULE_GUID // 14EEBB90-890A-43DB-AED1-5D3C4588A418 +("\x90\xBB\xEE\x14\x0A\x89\xDB\x43\xAE\xD1\x5D\x3C\x45\x88\xA4\x18", 16); +// Standard file system GUIDs +extern const UByteArray EFI_FIRMWARE_FILE_SYSTEM_GUID // 7A9354D9-0468-444A-81CE-0BF617D890DF +("\xD9\x54\x93\x7A\x68\x04\x4A\x44\x81\xCE\x0B\xF6\x17\xD8\x90\xDF", 16); +extern const UByteArray EFI_FIRMWARE_FILE_SYSTEM2_GUID // 8C8CE578-8A3D-4F1C-9935-896185C32DD3 +("\x78\xE5\x8C\x8C\x3D\x8A\x1C\x4F\x99\x35\x89\x61\x85\xC3\x2D\xD3", 16); +extern const UByteArray EFI_FIRMWARE_FILE_SYSTEM3_GUID // 5473C07A-3DCB-4DCA-BD6F-1E9689E7349A +("\x7A\xC0\x73\x54\xCB\x3D\xCA\x4D\xBD\x6F\x1E\x96\x89\xE7\x34\x9A", 16); +// Vendor-specific file system GUIDs +extern const UByteArray EFI_APPLE_IMMUTABLE_FV_GUID // 04ADEEAD-61FF-4D31-B6BA-64F8BF901F5A +("\xAD\xEE\xAD\x04\xFF\x61\x31\x4D\xB6\xBA\x64\xF8\xBF\x90\x1F\x5A", 16); +extern const UByteArray EFI_APPLE_AUTHENTICATION_FV_GUID // BD001B8C-6A71-487B-A14F-0C2A2DCF7A5D +("\x8C\x1B\x00\xBD\x71\x6A\x7B\x48\xA1\x4F\x0C\x2A\x2D\xCF\x7A\x5D", 16); +extern const UByteArray EFI_APPLE_MICROCODE_VOLUME_GUID // 153D2197-29BD-44DC-AC59-887F70E41A6B +("\x97\x21\x3D\x15\xBD\x29\xDC\x44\xAC\x59\x88\x7F\x70\xE4\x1A\x6B", 16); +extern const UByteArray EFI_INTEL_FILE_SYSTEM_GUID // AD3FFFFF-D28B-44C4-9F13-9EA98A97F9F0 +("\xFF\xFF\x3F\xAD\x8B\xD2\xC4\x44\x9F\x13\x9E\xA9\x8A\x97\xF9\xF0", 16); +extern const UByteArray EFI_INTEL_FILE_SYSTEM2_GUID // D6A1CD70-4B33-4994-A6EA-375F2CCC5437 +("\x70\xCD\xA1\xD6\x33\x4B\x94\x49\xA6\xEA\x37\x5F\x2C\xCC\x54\x37", 16); +extern const UByteArray EFI_SONY_FILE_SYSTEM_GUID // 4F494156-AED6-4D64-A537-B8A5557BCEEC +("\x56\x41\x49\x4F\xD6\xAE\x64\x4D\xA5\x37\xB8\xA5\x55\x7B\xCE\xEC", 16); +// PEI apriori file +extern const UByteArray EFI_PEI_APRIORI_FILE_GUID // 1B45CC0A-156A-428A-AF62-49864DA0E6E6 +("\x0A\xCC\x45\x1B\x6A\x15\x8A\x42\xAF\x62\x49\x86\x4D\xA0\xE6\xE6", 16); +// DXE apriori file +extern const UByteArray EFI_DXE_APRIORI_FILE_GUID // FC510EE7-FFDC-11D4-BD41-0080C73C8881 +("\xE7\x0E\x51\xFC\xDC\xFF\xD4\x11\xBD\x41\x00\x80\xC7\x3C\x88\x81", 16); +// Volume top file +extern const UByteArray EFI_FFS_VOLUME_TOP_FILE_GUID // 1BA0062E-C779-4582-8566-336AE8F78F09 +("\x2E\x06\xA0\x1B\x79\xC7\x82\x45\x85\x66\x33\x6A\xE8\xF7\x8F\x09", 16); +// Pad file GUID +extern const UByteArray EFI_FFS_PAD_FILE_GUID // E4536585-7909-4A60-B5C6-ECDEA6EBFB5 +("\x85\x65\x53\xE4\x09\x79\x60\x4A\xB5\xC6\xEC\xDE\xA6\xEB\xFB\x54", 16); +// AMI DXE core file +extern const UByteArray AMI_CORE_DXE_GUID // 5AE3F37E-4EAE-41AE-8240-35465B5E81EB +("\x7E\xF3\xE3\x5A\xAE\x4E\xAE\x41\x82\x40\x35\x46\x5B\x5E\x81\xEB", 16); +// EDK2 DXE core file +extern const UByteArray EFI_DXE_CORE_GUID // D6A2CB7F-6A18-4E2F-B43B-9920A733700A +("\x7F\xCB\xA2\xD6\x18\x6A\x2F\x4E\xB4\x3B\x99\x20\xA7\x33\x70\x0A", 16); +// GUIDs of GUID-defined sections +extern const UByteArray EFI_GUIDED_SECTION_CRC32 // FC1BCDB0-7D31-49AA-936A-A4600D9DD083 +("\xB0\xCD\x1B\xFC\x31\x7D\xAA\x49\x93\x6A\xA4\x60\x0D\x9D\xD0\x83", 16); +extern const UByteArray EFI_GUIDED_SECTION_TIANO // A31280AD-481E-41B6-95E8-127F4C984779 +("\xAD\x80\x12\xA3\x1E\x48\xB6\x41\x95\xE8\x12\x7F\x4C\x98\x47\x79", 16); +extern const UByteArray EFI_GUIDED_SECTION_LZMA // EE4E5898-3914-4259-9D6E-DC7BD79403CF +("\x98\x58\x4E\xEE\x14\x39\x59\x42\x9D\x6E\xDC\x7B\xD7\x94\x03\xCF", 16); +extern const UByteArray EFI_GUIDED_SECTION_LZMA_HP // 0ED85E23-F253-413F-A03C-901987B04397 +("\x23\x5E\xD8\x0E\x53\xF2\x3F\x41\xA0\x3C\x90\x19\x87\xB0\x43\x97", 16); +extern const UByteArray EFI_GUIDED_SECTION_LZMAF86 // D42AE6BD-1352-4BFB-909A-CA72A6EAE889 +("\xBD\xE6\x2A\xD4\x52\x13\xFB\x4B\x90\x9A\xCA\x72\xA6\xEA\xE8\x89", 16); +extern const UByteArray EFI_GUIDED_SECTION_GZIP // 1D301FE9-BE79-4353-91C2-D23BC959AE0C +("\xE9\x1F\x30\x1D\x79\xBE\x53\x43\x91\xC2\xD2\x3B\xC9\x59\xAE\x0C", 16); +extern const UByteArray EFI_FIRMWARE_CONTENTS_SIGNED_GUID // 0F9D89E8-9259-4F76-A5AF-0C89E34023DF +("\xE8\x89\x9D\x0F\x59\x92\x76\x4F\xA5\xAF\x0C\x89\xE3\x40\x23\xDF", 16); +extern const UByteArray EFI_CERT_TYPE_RSA2048_SHA256_GUID // A7717414-C616-4977-9420-844712A735BF + ("\x14\x74\x71\xA7\x16\xC6\x77\x49\x94\x20\x84\x47\x12\xA7\x35\xBF"); +extern const UByteArray EFI_HASH_ALGORITHM_SHA256_GUID // 51AA59DE-FDF2-4EA3-BC63-875FB7842EE9 +("\xde\x59\xAA\x51\xF2\xFD\xA3\x4E\xBC\x63\x87\x5F\xB7\x84\x2E\xE9"); + +// Protected range files +extern const UByteArray PROTECTED_RANGE_VENDOR_HASH_FILE_GUID_PHOENIX // 389CC6F2-1EA8-467B-AB8A-78E769AE2A15 +("\xF2\xC6\x9C\x38\xA8\x1E\x7B\x46\xAB\x8A\x78\xE7\x69\xAE\x2A\x15", 16); +extern const UByteArray PROTECTED_RANGE_VENDOR_HASH_FILE_GUID_AMI // CBC91F44-A4BC-4A5B-8696-703451D0B053 +("\x44\x1F\xC9\xCB\xBC\xA4\x5B\x4A\x86\x96\x70\x34\x51\xD0\xB0\x53", 16); + +const std::vector FFSv2Volumes({ EFI_FIRMWARE_FILE_SYSTEM_GUID, EFI_FIRMWARE_FILE_SYSTEM2_GUID, EFI_APPLE_AUTHENTICATION_FV_GUID, @@ -24,12 +115,9 @@ const UByteArray FFSv2VolumesInt[] = { EFI_INTEL_FILE_SYSTEM_GUID, EFI_INTEL_FILE_SYSTEM2_GUID, EFI_SONY_FILE_SYSTEM_GUID -}; -// This number must be updated if the array above is grown -#define FFSv2VolumesIntSize 7 -const std::vector FFSv2Volumes(FFSv2VolumesInt, FFSv2VolumesInt + FFSv2VolumesIntSize); -// Luckily, FFSv3Volumes now only has 1 element -const std::vector FFSv3Volumes(1, EFI_FIRMWARE_FILE_SYSTEM3_GUID); +}); + +const std::vector FFSv3Volumes({EFI_FIRMWARE_FILE_SYSTEM3_GUID}); const UINT8 ffsAlignmentTable[] = { 0, 4, 7, 9, 10, 12, 15, 16 }; @@ -118,8 +206,8 @@ UString fileTypeToUString(const UINT8 type) case EFI_FV_FILETYPE_MM_STANDALONE: return UString("MM standalone module"); case EFI_FV_FILETYPE_MM_CORE_STANDALONE: return UString("MM standalone core"); case EFI_FV_FILETYPE_PAD: return UString("Pad"); - default: return usprintf("Unknown %u", type); }; + return usprintf("Unknown %02Xh", type); } UString sectionTypeToUString(const UINT8 type) @@ -142,8 +230,8 @@ UString sectionTypeToUString(const UINT8 type) case EFI_SECTION_MM_DEPEX: return UString("MM dependency"); case INSYDE_SECTION_POSTCODE: return UString("Insyde postcode"); case PHOENIX_SECTION_POSTCODE: return UString("Phoenix postcode"); - default: return usprintf("Unknown %u", type); } + return usprintf("Unknown %02Xh", type); } UString bpdtEntryTypeToUString(const UINT16 type) @@ -179,8 +267,8 @@ UString bpdtEntryTypeToUString(const UINT16 type) case BPDT_ENTRY_TYPE_PCHC: return UString("PCHC"); case BPDT_ENTRY_TYPE_SAMF: return UString("SAMF"); case BPDT_ENTRY_TYPE_PPHY: return UString("PPHY"); - default: return usprintf("Unknown %u", type); } + return usprintf("Unknown %04Xh", type); } UString cpdExtensionTypeToUstring(const UINT32 type) @@ -218,6 +306,6 @@ UString cpdExtensionTypeToUstring(const UINT32 type) case CPD_EXT_TYPE_KEY_MANIFEST_EXT: return UString("Extended Key Manifest"); case CPD_EXT_TYPE_SIGNED_PACKAGE_INFO_EXT: return UString("Extended Signed Package Info"); case CPD_EXT_TYPE_SPS_PLATFORM_ID: return UString("SPS Platform ID"); - default: return usprintf("Unknown %u", type); } + return usprintf("Unknown %08Xh", type); } diff --git a/common/ffs.h b/common/ffs.h index 1d02efc..fdfbf46 100644 --- a/common/ffs.h +++ b/common/ffs.h @@ -45,24 +45,19 @@ typedef struct EFI_CAPSULE_HEADER_ { #define EFI_CAPSULE_HEADER_FLAG_POPULATE_SYSTEM_TABLE 0x00020000 // Standard FMP capsule GUID -const UByteArray EFI_FMP_CAPSULE_GUID // 6DCBD5ED-E82D-4C44-BDA1-7194199AD92A -("\xED\xD5\xCB\x6D\x2D\xE8\x44\x4C\xBD\xA1\x71\x94\x19\x9A\xD9\x2A", 16); +extern const UByteArray EFI_FMP_CAPSULE_GUID; // 6DCBD5ED-E82D-4C44-BDA1-7194199AD92A // Standard EFI capsule GUID -const UByteArray EFI_CAPSULE_GUID // 3B6686BD-0D76-4030-B70E-B5519E2FC5A0 -("\xBD\x86\x66\x3B\x76\x0D\x30\x40\xB7\x0E\xB5\x51\x9E\x2F\xC5\xA0", 16); +extern const UByteArray EFI_CAPSULE_GUID; // 3B6686BD-0D76-4030-B70E-B5519E2FC5A0 // Intel capsule GUID -const UByteArray INTEL_CAPSULE_GUID -("\xB9\x82\x91\x53\xB5\xAB\x91\x43\xB6\x9A\xE3\xA9\x43\xF7\x2F\xCC", 16); +extern const UByteArray INTEL_CAPSULE_GUID; // 539182B9-ABB5-4391-B69A-E3A943F72FCC // Lenovo capsule GUID -const UByteArray LENOVO_CAPSULE_GUID -("\xD3\xAF\x0B\xE2\x14\x99\x4F\x4F\x95\x37\x31\x29\xE0\x90\xEB\x3C", 16); +extern const UByteArray LENOVO_CAPSULE_GUID; // E20BAFD3-9914-4F4F-9537-3129E090EB3C // Another Lenovo capsule GUID -const UByteArray LENOVO2_CAPSULE_GUID -("\x76\xFE\xB5\x25\x43\x82\x5C\x4A\xA9\xBD\x7E\xE3\x24\x61\x98\xB5", 16); +extern const UByteArray LENOVO2_CAPSULE_GUID; // 25B5FE76-8243-4A5C-A9BD-7EE3246198B5 // Toshiba EFI Capsule header typedef struct TOSHIBA_CAPSULE_HEADER_ { @@ -73,8 +68,7 @@ typedef struct TOSHIBA_CAPSULE_HEADER_ { } TOSHIBA_CAPSULE_HEADER; // Toshiba capsule GUID -const UByteArray TOSHIBA_CAPSULE_GUID // 3BE07062-1D51-45D2-832B-F093257ED461 -("\x62\x70\xE0\x3B\x51\x1D\xD2\x45\x83\x2B\xF0\x93\x25\x7E\xD4\x61", 16); +extern const UByteArray TOSHIBA_CAPSULE_GUID; // 3BE07062-1D51-45D2-832B-F093257ED461 // AMI Aptio extended capsule header typedef struct APTIO_CAPSULE_HEADER_ { @@ -86,12 +80,10 @@ typedef struct APTIO_CAPSULE_HEADER_ { } APTIO_CAPSULE_HEADER; // AMI Aptio signed extended capsule GUID -const UByteArray APTIO_SIGNED_CAPSULE_GUID -("\x8B\xA6\x3C\x4A\x23\x77\xFB\x48\x80\x3D\x57\x8C\xC1\xFE\xC4\x4D", 16); +extern const UByteArray APTIO_SIGNED_CAPSULE_GUID; // 4A3CA68B-7723-48FB-803D-578CC1FEC44D // AMI Aptio unsigned extended capsule GUID -const UByteArray APTIO_UNSIGNED_CAPSULE_GUID -("\x90\xBB\xEE\x14\x0A\x89\xDB\x43\xAE\xD1\x5D\x3C\x45\x88\xA4\x18", 16); +extern const UByteArray APTIO_UNSIGNED_CAPSULE_GUID; // 14EEBB90-890A-43DB-AED1-5D3C4588A418 //***************************************************************************** // EFI Firmware Volume @@ -119,34 +111,25 @@ typedef struct EFI_FIRMWARE_VOLUME_HEADER_ { } EFI_FIRMWARE_VOLUME_HEADER; // Standard file system GUIDs -const UByteArray EFI_FIRMWARE_FILE_SYSTEM_GUID // 7A9354D9-0468-444A-81CE-0BF617D890DF -("\xD9\x54\x93\x7A\x68\x04\x4A\x44\x81\xCE\x0B\xF6\x17\xD8\x90\xDF", 16); +extern const UByteArray EFI_FIRMWARE_FILE_SYSTEM_GUID; // 7A9354D9-0468-444A-81CE-0BF617D890DF -const UByteArray EFI_FIRMWARE_FILE_SYSTEM2_GUID // 8C8CE578-8A3D-4F1C-9935-896185C32DD3 -("\x78\xE5\x8C\x8C\x3D\x8A\x1C\x4F\x99\x35\x89\x61\x85\xC3\x2D\xD3", 16); +extern const UByteArray EFI_FIRMWARE_FILE_SYSTEM2_GUID; // 8C8CE578-8A3D-4F1C-9935-896185C32DD3 -const UByteArray EFI_FIRMWARE_FILE_SYSTEM3_GUID // 5473C07A-3DCB-4DCA-BD6F-1E9689E7349A -("\x7A\xC0\x73\x54\xCB\x3D\xCA\x4D\xBD\x6F\x1E\x96\x89\xE7\x34\x9A", 16); +extern const UByteArray EFI_FIRMWARE_FILE_SYSTEM3_GUID; // 5473C07A-3DCB-4DCA-BD6F-1E9689E7349A // Vendor-specific file system GUIDs -const UByteArray EFI_APPLE_IMMUTABLE_FV_GUID // 04ADEEAD-61FF-4D31-B6BA-64F8BF901F5A -("\xAD\xEE\xAD\x04\xFF\x61\x31\x4D\xB6\xBA\x64\xF8\xBF\x90\x1F\x5A", 16); +extern const UByteArray EFI_APPLE_IMMUTABLE_FV_GUID; // 04ADEEAD-61FF-4D31-B6BA-64F8BF901F5A -const UByteArray EFI_APPLE_AUTHENTICATION_FV_GUID // BD001B8C-6A71-487B-A14F-0C2A2DCF7A5D -("\x8C\x1B\x00\xBD\x71\x6A\x7B\x48\xA1\x4F\x0C\x2A\x2D\xCF\x7A\x5D", 16); +extern const UByteArray EFI_APPLE_AUTHENTICATION_FV_GUID; // BD001B8C-6A71-487B-A14F-0C2A2DCF7A5D -const UByteArray EFI_APPLE_MICROCODE_VOLUME_GUID // 153D2197-29BD-44DC-AC59-887F70E41A6B -("\x97\x21\x3D\x15\xBD\x29\xDC\x44\xAC\x59\x88\x7F\x70\xE4\x1A\x6B", 16); +extern const UByteArray EFI_APPLE_MICROCODE_VOLUME_GUID; // 153D2197-29BD-44DC-AC59-887F70E41A6B #define EFI_APPLE_MICROCODE_VOLUME_HEADER_SIZE 0x100 -const UByteArray EFI_INTEL_FILE_SYSTEM_GUID // AD3FFFFF-D28B-44C4-9F13-9EA98A97F9F0 -("\xFF\xFF\x3F\xAD\x8B\xD2\xC4\x44\x9F\x13\x9E\xA9\x8A\x97\xF9\xF0", 16); +extern const UByteArray EFI_INTEL_FILE_SYSTEM_GUID; // AD3FFFFF-D28B-44C4-9F13-9EA98A97F9F0 -const UByteArray EFI_INTEL_FILE_SYSTEM2_GUID // D6A1CD70-4B33-4994-A6EA-375F2CCC5437 -("\x70\xCD\xA1\xD6\x33\x4B\x94\x49\xA6\xEA\x37\x5F\x2C\xCC\x54\x37", 16); +extern const UByteArray EFI_INTEL_FILE_SYSTEM2_GUID; // D6A1CD70-4B33-4994-A6EA-375F2CCC5437 -const UByteArray EFI_SONY_FILE_SYSTEM_GUID // 4F494156-AED6-4D64-A537-B8A5557BCEEC -("\x56\x41\x49\x4F\xD6\xAE\x64\x4D\xA5\x37\xB8\xA5\x55\x7B\xCE\xEC", 16); +extern const UByteArray EFI_SONY_FILE_SYSTEM_GUID; // 4F494156-AED6-4D64-A537-B8A5557BCEEC // Vector of volume GUIDs with FFSv2-compatible files extern const std::vector FFSv2Volumes; @@ -357,32 +340,22 @@ extern const UINT8 ffsAlignment2Table[]; #define EFI_FILE_ERASE_POLARITY 0x80 // Defined as "all other bits must be set to ERASE_POLARITY" in UEFI PI // PEI apriori file -const UByteArray EFI_PEI_APRIORI_FILE_GUID -("\x0A\xCC\x45\x1B\x6A\x15\x8A\x42\xAF\x62\x49\x86\x4D\xA0\xE6\xE6", 16); +extern const UByteArray EFI_PEI_APRIORI_FILE_GUID; // 1B45CC0A-156A-428A-AF62-49864DA0E6E6 // DXE apriori file -const UByteArray EFI_DXE_APRIORI_FILE_GUID -("\xE7\x0E\x51\xFC\xDC\xFF\xD4\x11\xBD\x41\x00\x80\xC7\x3C\x88\x81", 16); +extern const UByteArray EFI_DXE_APRIORI_FILE_GUID; // FC510EE7-FFDC-11D4-BD41-0080C73C8881 // Volume top file -const UByteArray EFI_FFS_VOLUME_TOP_FILE_GUID -("\x2E\x06\xA0\x1B\x79\xC7\x82\x45\x85\x66\x33\x6A\xE8\xF7\x8F\x09", 16); +extern const UByteArray EFI_FFS_VOLUME_TOP_FILE_GUID; // 1BA0062E-C779-4582-8566-336AE8F78F09 -// Pad file GUID -const UByteArray EFI_FFS_PAD_FILE_GUID -("\x85\x65\x53\xE4\x09\x79\x60\x4A\xB5\xC6\xEC\xDE\xA6\xEB\xFB\x54", 16); +// AMI pad file GUID +extern const UByteArray EFI_FFS_PAD_FILE_GUID; // E4536585-7909-4A60-B5C6-ECDEA6EBFB5 // AMI DXE core file -const UByteArray AMI_CORE_DXE_GUID // 5AE3F37E-4EAE-41AE-8240-35465B5E81EB -("\x7E\xF3\xE3\x5A\xAE\x4E\xAE\x41\x82\x40\x35\x46\x5B\x5E\x81\xEB", 16); +extern const UByteArray AMI_CORE_DXE_GUID; // 5AE3F37E-4EAE-41AE-8240-35465B5E81EB -// EDK2 DXE code file -const UByteArray EFI_DXE_CORE_GUID // D6A2CB7F-6A18-4E2F-B43B-9920A733700A -("\x7F\xCB\xA2\xD6\x18\x6A\x2F\x4E\xB4\x3B\x99\x20\xA7\x33\x70\x0A", 16); - -// TXT ACM -const UByteArray EFI_TXT_ACM_GUID // 2D27C618-7DCD-41F5-BB10-21166BE7E143 -("\x18\xC6\x27\x2D\xCD\x7D\xF5\x41\xBB\x10\x21\x16\x6B\xE7\xE1\x43", 16); +// EDK2 DXE core file +extern const UByteArray EFI_DXE_CORE_GUID; // D6A2CB7F-6A18-4E2F-B43B-9920A733700A // FFS size conversion routines extern VOID uint32ToUint24(UINT32 size, UINT8* ffsSize); @@ -474,26 +447,13 @@ typedef struct EFI_GUID_DEFINED_SECTION_APPLE_ { #define EFI_GUIDED_SECTION_AUTH_STATUS_VALID 0x02 // GUIDs of GUID-defined sections -const UByteArray EFI_GUIDED_SECTION_CRC32 // FC1BCDB0-7D31-49AA-936A-A4600D9DD083 -("\xB0\xCD\x1B\xFC\x31\x7D\xAA\x49\x93\x6A\xA4\x60\x0D\x9D\xD0\x83", 16); - -const UByteArray EFI_GUIDED_SECTION_TIANO // A31280AD-481E-41B6-95E8-127F4C984779 -("\xAD\x80\x12\xA3\x1E\x48\xB6\x41\x95\xE8\x12\x7F\x4C\x98\x47\x79", 16); - -const UByteArray EFI_GUIDED_SECTION_LZMA // EE4E5898-3914-4259-9D6E-DC7BD79403CF -("\x98\x58\x4E\xEE\x14\x39\x59\x42\x9D\x6E\xDC\x7B\xD7\x94\x03\xCF", 16); - -const UByteArray EFI_GUIDED_SECTION_LZMA_HP // 0ED85E23-F253-413F-A03C-901987B04397 -("\x23\x5E\xD8\x0E\x53\xF2\x3F\x41\xA0\x3C\x90\x19\x87\xB0\x43\x97", 16); - -const UByteArray EFI_GUIDED_SECTION_LZMAF86 // D42AE6BD-1352-4BFB-909A-CA72A6EAE889 -("\xBD\xE6\x2A\xD4\x52\x13\xFB\x4B\x90\x9A\xCA\x72\xA6\xEA\xE8\x89", 16); - -const UByteArray EFI_GUIDED_SECTION_GZIP // 1D301FE9-BE79-4353-91C2-D23BC959AE0C -("\xE9\x1F\x30\x1D\x79\xBE\x53\x43\x91\xC2\xD2\x3B\xC9\x59\xAE\x0C", 16); - -const UByteArray EFI_FIRMWARE_CONTENTS_SIGNED_GUID // 0F9D89E8-9259-4F76-A5AF-0C89E34023DF -("\xE8\x89\x9D\x0F\x59\x92\x76\x4F\xA5\xAF\x0C\x89\xE3\x40\x23\xDF", 16); +extern const UByteArray EFI_GUIDED_SECTION_CRC32; // FC1BCDB0-7D31-49AA-936A-A4600D9DD083 +extern const UByteArray EFI_GUIDED_SECTION_TIANO; // A31280AD-481E-41B6-95E8-127F4C984779 +extern const UByteArray EFI_GUIDED_SECTION_LZMA; // EE4E5898-3914-4259-9D6E-DC7BD79403CF +extern const UByteArray EFI_GUIDED_SECTION_LZMA_HP; // 0ED85E23-F253-413F-A03C-901987B04397 +extern const UByteArray EFI_GUIDED_SECTION_LZMAF86; // D42AE6BD-1352-4BFB-909A-CA72A6EAE889 +extern const UByteArray EFI_GUIDED_SECTION_GZIP; // 1D301FE9-BE79-4353-91C2-D23BC959AE0C +extern const UByteArray EFI_FIRMWARE_CONTENTS_SIGNED_GUID; // 0F9D89E8-9259-4F76-A5AF-0C89E34023DF //#define WIN_CERT_TYPE_PKCS_SIGNED_DATA 0x0002 #define WIN_CERT_TYPE_EFI_GUID 0x0EF1 @@ -512,8 +472,7 @@ typedef struct WIN_CERTIFICATE_UEFI_GUID_ { } WIN_CERTIFICATE_UEFI_GUID; // WIN_CERTIFICATE_UEFI_GUID.CertType -const UByteArray EFI_CERT_TYPE_RSA2048_SHA256_GUID // A7717414-C616-4977-9420-844712A735BF -("\x14\x74\x71\xA7\x16\xC6\x77\x49\x94\x20\x84\x47\x12\xA7\x35\xBF"); +extern const UByteArray EFI_CERT_TYPE_RSA2048_SHA256_GUID; // A7717414-C616-4977-9420-844712A735BF // WIN_CERTIFICATE_UEFI_GUID.CertData typedef struct EFI_CERT_BLOCK_RSA2048_SHA256_ { @@ -522,8 +481,7 @@ typedef struct EFI_CERT_BLOCK_RSA2048_SHA256_ { UINT8 Signature[256]; } EFI_CERT_BLOCK_RSA2048_SHA256; -const UByteArray EFI_HASH_ALGORITHM_SHA256_GUID // 51aa59de-fdf2-4ea3-bc63-875fb7842ee9 -("\xde\x59\xAA\x51\xF2\xFD\xA3\x4E\xBC\x63\x87\x5F\xB7\x84\x2E\xE9"); +extern const UByteArray EFI_HASH_ALGORITHM_SHA256_GUID; // 51AA59DE-FDF2-4EA3-BC63-875FB7842EE9 // Version section typedef struct EFI_VERSION_SECTION_ { @@ -811,6 +769,49 @@ typedef struct CPD_EXT_IFWI_PARTITION_MANIFEST_ { UINT8 Reserved[4]; } CPD_EXT_IFWI_PARTITION_MANIFEST; +//***************************************************************************** +// Protected range +//***************************************************************************** +extern const UByteArray PROTECTED_RANGE_VENDOR_HASH_FILE_GUID_PHOENIX; // 389CC6F2-1EA8-467B-AB8A-78E769AE2A15 + +#define BG_VENDOR_HASH_FILE_SIGNATURE_PHOENIX 0x4C42544853414824ULL // '$HASHTBL' + +extern const UByteArray PROTECTED_RANGE_VENDOR_HASH_FILE_GUID_AMI; // CBC91F44-A4BC-4A5B-8696-703451D0B053 + +typedef struct BG_VENDOR_HASH_FILE_ENTRY +{ + UINT8 Hash[SHA256_HASH_SIZE]; + UINT32 Base; + UINT32 Size; +} PROTECTED_RANGE_VENDOR_HASH_FILE_ENTRY; + +typedef struct PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_PHOENIX_ +{ + UINT64 Signature; + UINT32 NumEntries; + //BG_VENDOR_HASH_FILE_ENTRY Entries[]; +} PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_PHOENIX; + +typedef struct PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V1_ +{ + UINT8 Hash[SHA256_HASH_SIZE]; + UINT32 Size; + // Base is derived from flash map, will be detected as root volume with DXE core +} PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V1; + +typedef struct PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V2_ +{ + BG_VENDOR_HASH_FILE_ENTRY Hash0; + BG_VENDOR_HASH_FILE_ENTRY Hash1; +} PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V2; + +typedef struct PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V3_ +{ + UINT8 Hash[SHA256_HASH_SIZE]; + // UINT32 Base[SOME_HARDCODED_N] + // UINT32 Size[SOME_HARDCODED_N]; +} PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V3; + // Restore previous packing rules #pragma pack(pop) diff --git a/common/ffsparser.cpp b/common/ffsparser.cpp index 9e33795..2e97af5 100644 --- a/common/ffsparser.cpp +++ b/common/ffsparser.cpp @@ -20,7 +20,7 @@ #include "ffs.h" #include "gbe.h" #include "me.h" -#include "fit.h" +#include "intel_fit.h" #include "nvram.h" #include "peimage.h" #include "parsingdata.h" @@ -29,7 +29,11 @@ #include "nvramparser.h" #include "meparser.h" -#include "uinttypes.h" +#include "fitparser.h" + +#include "digest/sha1.h" +#include "digest/sha2.h" +#include "digest/sm3.h" #ifndef QT_CORE_LIB namespace Qt { @@ -70,8 +74,8 @@ struct CPD_PARTITION_INFO { // Constructor FfsParser::FfsParser(TreeModel* treeModel) : model(treeModel), -imageBase(0), addressDiff(0x100000000ULL), -bgAcmFound(false), bgKeyManifestFound(false), bgBootPolicyFound(false), bgProtectedRegionsBase(0) { +imageBase(0), addressDiff(0x100000000ULL), protectedRegionsBase(0) { + fitParser = new FitParser(treeModel, this); nvramParser = new NvramParser(treeModel, this); meParser = new MeParser(treeModel, this); } @@ -80,18 +84,32 @@ bgAcmFound(false), bgKeyManifestFound(false), bgBootPolicyFound(false), bgProtec FfsParser::~FfsParser() { delete nvramParser; delete meParser; + delete fitParser; } // Obtain parser messages std::vector > FfsParser::getMessages() const { std::vector > meVector = meParser->getMessages(); std::vector > nvramVector = nvramParser->getMessages(); + std::vector > fitVector = fitParser->getMessages(); std::vector > resultVector = messagesVector; resultVector.insert(resultVector.end(), meVector.begin(), meVector.end()); - resultVector.insert(resultVector.end(), nvramVector.begin(), nvramVector.end()); + resultVector.insert(resultVector.end(), nvramVector.begin(), nvramVector.end());\ + resultVector.insert(resultVector.end(), fitVector.begin(), fitVector.end()); return resultVector; } +// Obtain FIT table from FIT parser +std::vector, UModelIndex> > FfsParser::getFitTable() const +{ + return fitParser->getFitTable(); +} + +// Obtain security info from FIT parser +UString FfsParser::getSecurityInfo() const { + return securityInfo + fitParser->getSecurityInfo(); +} + // Firmware image parsing functions USTATUS FfsParser::parse(const UByteArray & buffer) { @@ -101,21 +119,11 @@ USTATUS FfsParser::parse(const UByteArray & buffer) openedImage = buffer; imageBase = 0; addressDiff = 0x100000000ULL; - bgAcmFound = false; - bgKeyManifestFound = false; - bgBootPolicyFound = false; - bgProtectedRegionsBase = 0; - lastVtf = UModelIndex(); - fitTable.clear(); + protectedRegionsBase = 0; securityInfo = ""; - bgAcmFound = false; - bgKeyManifestFound = false; - bgBootPolicyFound = false; - bgKmHash = UByteArray(); - bgBpHash = UByteArray(); - bgBpDigest = UByteArray(); - bgProtectedRanges.clear(); - bgDxeCoreIndex = UModelIndex(); + protectedRanges.clear(); + lastVtf = UModelIndex(); + dxeCore = UModelIndex(); // Parse input buffer USTATUS result = performFirstPass(buffer, root); @@ -136,7 +144,7 @@ USTATUS FfsParser::performFirstPass(const UByteArray & buffer, UModelIndex & ind { // Sanity check if (buffer.isEmpty()) { - return EFI_INVALID_PARAMETER; + return U_INVALID_PARAMETER; } USTATUS result; @@ -167,7 +175,7 @@ USTATUS FfsParser::parseGenericImage(const UByteArray & buffer, const UINT32 loc index = model->addItem(localOffset, Types::Image, Subtypes::UefiImage, name, UString(), info, UByteArray(), buffer, UByteArray(), Fixed, parent); // Parse the image as raw area - bgProtectedRegionsBase = imageBase = model->base(parent) + localOffset; + protectedRegionsBase = imageBase = model->base(parent) + localOffset; return parseRawArea(index); } @@ -514,7 +522,7 @@ USTATUS FfsParser::parseIntelImage(const UByteArray & intelImage, const UINT32 l // Add offsets of actual regions for (size_t i = 0; i < regions.size(); i++) { if (regions[i].type != Subtypes::ZeroPadding && regions[i].type != Subtypes::OnePadding && regions[i].type != Subtypes::DataPadding) - info += UString("\n") + itemSubtypeToUString(Types::Region, regions[i].type) + info += "\n" + itemSubtypeToUString(Types::Region, regions[i].type) + usprintf(" region offset: %Xh", regions[i].offset + localOffset); } @@ -597,7 +605,7 @@ USTATUS FfsParser::parseIntelImage(const UByteArray & intelImage, const UINT32 l info += usprintf("\n%02X%02X%02X (", vsccTableEntry->VendorId, vsccTableEntry->DeviceId0, vsccTableEntry->DeviceId1) + jedecId + UString(")"); - if (jedecId == UString("Unknown")) { + if (jedecId.startsWith("Unknown")) { msg(usprintf("%s: SPI flash with unknown JEDEC ID %02X%02X%02X found in VSCC table", __FUNCTION__, vsccTableEntry->VendorId, vsccTableEntry->DeviceId0, vsccTableEntry->DeviceId1), index); } @@ -714,11 +722,15 @@ USTATUS FfsParser::parseMeRegion(const UByteArray & me, const UINT32 localOffset } else { // Search for new signature - INT32 versionOffset = (INT32)me.indexOf(ME_VERSION_SIGNATURE2); - if (versionOffset < 0){ // New signature not found + UINT32 sig2Value = ME_VERSION_SIGNATURE2; + UByteArray sig2((const char*)&sig2Value, sizeof(sig2Value)); + INT32 versionOffset = (INT32)me.indexOf(sig2); + if (versionOffset < 0) { // New signature not found // Search for old signature - versionOffset = (INT32)me.indexOf(ME_VERSION_SIGNATURE); - if (versionOffset < 0){ + UINT32 sigValue = ME_VERSION_SIGNATURE; + UByteArray sig((const char*)&sigValue, sizeof(sigValue)); + versionOffset = (INT32)me.indexOf(sig); + if (versionOffset < 0) { info += ("\nVersion: unknown"); versionFound = false; } @@ -865,7 +877,7 @@ USTATUS FfsParser::parseRawArea(const UModelIndex & index) // Set base of protected regions to be the first volume if (model->type(index) == Types::Region && model->subtype(index) == Subtypes::BiosRegion) { - bgProtectedRegionsBase = (UINT64)model->base(index) + prevItemOffset; + protectedRegionsBase = (UINT64)model->base(index) + prevItemOffset; } // First item is not at the beginning of this raw area @@ -1795,8 +1807,8 @@ USTATUS FfsParser::parseFileHeader(const UByteArray & file, const UINT32 localOf } // Override first DXE core index, if needed - if (isDxeCore && !bgDxeCoreIndex.isValid()) { - bgDxeCoreIndex = index; + if (isDxeCore && !dxeCore.isValid()) { + dxeCore = index; } // Show messages @@ -1874,7 +1886,7 @@ USTATUS FfsParser::parseFileBody(const UModelIndex & index) return nvramParser->parseNvarStore(index); } // Parse vendor hash file - else if (fileGuid == BG_VENDOR_HASH_FILE_GUID_PHOENIX) { + else if (fileGuid == PROTECTED_RANGE_VENDOR_HASH_FILE_GUID_PHOENIX) { return parseVendorHashFile(fileGuid, index); } @@ -2998,7 +3010,7 @@ USTATUS FfsParser::parseAprioriRawSection(const UByteArray & body, UString & par if (count > 0) { for (UINT32 i = 0; i < count; i++) { const EFI_GUID* guid = (const EFI_GUID*)body.constData() + i; - parsed += UString("\n") + guidToUString(readUnaligned(guid)); + parsed += "\n" + guidToUString(readUnaligned(guid)); } } @@ -3014,7 +3026,7 @@ USTATUS FfsParser::parseRawSectionBody(const UModelIndex & index) // Check for apriori file UModelIndex parentFile = model->findParentOfType(index, Types::File); if (!parentFile.isValid()) - return U_INVALID_FILE; //TODO: better return code + return U_INVALID_RAW_AREA; // Get parent file parsing data UByteArray parentFileGuid(model->header(parentFile).constData(), sizeof(EFI_GUID)); @@ -3044,7 +3056,7 @@ USTATUS FfsParser::parseRawSectionBody(const UModelIndex & index) // Parse NVAR area return nvramParser->parseNvarStore(index); } - else if (parentFileGuid == BG_VENDOR_HASH_FILE_GUID_AMI) { // AMI vendor hash file + else if (parentFileGuid == PROTECTED_RANGE_VENDOR_HASH_FILE_GUID_AMI) { // AMI vendor hash file // Parse AMI vendor hash file return parseVendorHashFile(parentFileGuid, index); } @@ -3209,7 +3221,7 @@ USTATUS FfsParser::performSecondPass(const UModelIndex & index) parseResetVectorData(); // Find and parse FIT - parseFit(index); + fitParser->parseFit(index); // Check protected ranges checkProtectedRanges(index); @@ -3237,8 +3249,8 @@ USTATUS FfsParser::parseResetVectorData() UString info = usprintf("\nAP entry vector: %02X %02X %02X %02X %02X %02X %02X %02X\n" "Reset vector: %02X %02X %02X %02X %02X %02X %02X %02X\n" "PEI core entry point: %08Xh\n" - "AP startup segment: %08X\n" - "BootFV base address: %08X\n", + "AP startup segment: %08Xh\n" + "BootFV base address: %08Xh\n", resetVectorData->ApEntryVector[0], resetVectorData->ApEntryVector[1], resetVectorData->ApEntryVector[2], resetVectorData->ApEntryVector[3], resetVectorData->ApEntryVector[4], resetVectorData->ApEntryVector[5], resetVectorData->ApEntryVector[6], resetVectorData->ApEntryVector[7], resetVectorData->ResetVector[0], resetVectorData->ResetVector[1], resetVectorData->ResetVector[2], resetVectorData->ResetVector[3], @@ -3254,8 +3266,9 @@ USTATUS FfsParser::parseResetVectorData() USTATUS FfsParser::checkTeImageBase(const UModelIndex & index) { // Sanity check - if (!index.isValid()) - return U_SUCCESS; + if (!index.isValid()) { + return U_INVALID_PARAMETER; + } // Determine relocation type of uncompressed TE image sections if (model->compressed(index) == false @@ -3336,11 +3349,11 @@ USTATUS FfsParser::addInfoRecursive(const UModelIndex & index) if (address <= 0xFFFFFFFFUL) { UINT32 headerSize = (UINT32)model->header(index).size(); if (headerSize) { - model->addInfo(index, usprintf("Data address: %08llXh\n", (unsigned long long)address + headerSize),false); - model->addInfo(index, usprintf("Header address: %08llXh\n", (unsigned long long)address), false); + model->addInfo(index, usprintf("Data address: %08Xh\n", (UINT32)address + headerSize),false); + model->addInfo(index, usprintf("Header address: %08Xh\n", (UINT32)address), false); } else { - model->addInfo(index, usprintf("Address: %08llXh\n", (unsigned long long)address), false); + model->addInfo(index, usprintf("Address: %08Xh\n", (UINT32)address), false); } } // Add base @@ -3366,17 +3379,16 @@ USTATUS FfsParser::checkProtectedRanges(const UModelIndex & index) UByteArray protectedParts; bool bgProtectedRangeFound = false; try { - for (UINT32 i = 0; i < (UINT32)bgProtectedRanges.size(); i++) { - if (bgProtectedRanges[i].Type == BG_PROTECTED_RANGE_INTEL_BOOT_GUARD_IBB && bgProtectedRanges[i].Size > 0) { + for (UINT32 i = 0; i < (UINT32)protectedRanges.size(); i++) { + if (protectedRanges[i].Type == PROTECTED_RANGE_INTEL_BOOT_GUARD_IBB && protectedRanges[i].Size > 0) { bgProtectedRangeFound = true; - if ((UINT64)bgProtectedRanges[i].Offset >= addressDiff) { - bgProtectedRanges[i].Offset -= (UINT32)addressDiff; + if ((UINT64)protectedRanges[i].Offset >= addressDiff) { + protectedRanges[i].Offset -= (UINT32)addressDiff; } else { - // TODO: Explore this. - msg(usprintf("%s: suspicious BG protection offset", __FUNCTION__), index); + msg(usprintf("%s: suspicious protected range offset", __FUNCTION__), index); } - protectedParts += openedImage.mid(bgProtectedRanges[i].Offset, bgProtectedRanges[i].Size); - markProtectedRangeRecursive(index, bgProtectedRanges[i]); + protectedParts += openedImage.mid(protectedRanges[i].Offset, protectedRanges[i].Size); + markProtectedRangeRecursive(index, protectedRanges[i]); } } } catch (...) { @@ -3384,138 +3396,216 @@ USTATUS FfsParser::checkProtectedRanges(const UModelIndex & index) } if (bgProtectedRangeFound) { - UByteArray digest(SHA256_DIGEST_SIZE, '\x00'); - sha256(protectedParts.constData(), protectedParts.size(), digest.data()); - - if (digest != bgBpDigest) { - msg(usprintf("%s: BG-protected ranges hash mismatch, opened image may refuse to boot", __FUNCTION__), index); + UINT8 digest[SHA512_HASH_SIZE] = {}; + UString digestString; + UString ibbDigests; + // SHA1 + digestString = ""; + sha1(protectedParts.constData(), protectedParts.size(), digest); + for (UINT8 i = 0; i < SHA1_HASH_SIZE; i++) { + digestString += usprintf("%02X", digest[i]); } - } - else if (bgBootPolicyFound) { - msg(usprintf("%s: BootPolicy doesn't define any BG-protected ranges", __FUNCTION__), index); + ibbDigests += UString("Computed IBB Hash (SHA1): ") + digestString + "\n"; + // SHA256 + digestString = ""; + sha256(protectedParts.constData(), protectedParts.size(), digest); + for (UINT8 i = 0; i < SHA256_HASH_SIZE; i++) { + digestString += usprintf("%02X", digest[i]); + } + ibbDigests += UString("Computed IBB Hash (SHA256): ") + digestString + "\n"; + // SHA384 + digestString = ""; + sha384(protectedParts.constData(), protectedParts.size(), digest); + for (UINT8 i = 0; i < SHA384_HASH_SIZE; i++) { + digestString += usprintf("%02X", digest[i]); + } + ibbDigests += UString("Computed IBB Hash (SHA384): ") + digestString + "\n"; + // SHA512 + digestString = ""; + sha512(protectedParts.constData(), protectedParts.size(), digest); + for (UINT8 i = 0; i < SHA512_HASH_SIZE; i++) { + digestString += usprintf("%02X", digest[i]); + } + ibbDigests += UString("Computed IBB Hash (SHA512): ") + digestString + "\n"; + // SM3 + digestString = ""; + sm3(protectedParts.constData(), protectedParts.size(), digest); + for (UINT8 i = 0; i < SM3_HASH_SIZE; i++) { + digestString += usprintf("%02X", digest[i]); + } + ibbDigests += UString("Computed IBB Hash (SM3): ") + digestString + "\n"; + + securityInfo += ibbDigests + "\n"; } // Calculate digests for vendor-protected ranges - for (UINT32 i = 0; i < (UINT32)bgProtectedRanges.size(); i++) { - if (bgProtectedRanges[i].Type == BG_PROTECTED_RANGE_VENDOR_HASH_AMI_OLD - && bgProtectedRanges[i].Size != 0 && bgProtectedRanges[i].Size != 0xFFFFFFFF) { - if (!bgDxeCoreIndex.isValid()) { + for (UINT32 i = 0; i < (UINT32)protectedRanges.size(); i++) { + if (protectedRanges[i].Type == PROTECTED_RANGE_VENDOR_HASH_AMI_V1) { + if (!dxeCore.isValid()) { msg(usprintf("%s: can't determine DXE volume offset, old AMI protected range hash can't be checked", __FUNCTION__), index); } else { // Offset will be determined as the offset of root volume with first DXE core - UModelIndex dxeRootVolumeIndex = model->findLastParentOfType(bgDxeCoreIndex, Types::Volume); + UModelIndex dxeRootVolumeIndex = model->findLastParentOfType(dxeCore, Types::Volume); if (!dxeRootVolumeIndex.isValid()) { msg(usprintf("%s: can't determine DXE volume offset, old AMI protected range hash can't be checked", __FUNCTION__), index); } else { - bgProtectedRanges[i].Offset = model->base(dxeRootVolumeIndex); - protectedParts = openedImage.mid(bgProtectedRanges[i].Offset, bgProtectedRanges[i].Size); + protectedRanges[i].Offset = model->base(dxeRootVolumeIndex); + protectedParts = openedImage.mid(protectedRanges[i].Offset, protectedRanges[i].Size); - UByteArray digest(SHA256_DIGEST_SIZE, '\x00'); + UByteArray digest(SHA256_HASH_SIZE, '\x00'); sha256(protectedParts.constData(), protectedParts.size(), digest.data()); - if (digest != bgProtectedRanges[i].Hash) { + if (digest != protectedRanges[i].Hash) { msg(usprintf("%s: old AMI protected range [%Xh:%Xh] hash mismatch, opened image may refuse to boot", __FUNCTION__, - bgProtectedRanges[i].Offset, bgProtectedRanges[i].Offset + bgProtectedRanges[i].Size), - model->findByBase(bgProtectedRanges[i].Offset)); + protectedRanges[i].Offset, protectedRanges[i].Offset + protectedRanges[i].Size), + model->findByBase(protectedRanges[i].Offset)); } - markProtectedRangeRecursive(index, bgProtectedRanges[i]); + markProtectedRangeRecursive(index, protectedRanges[i]); } } } - else if (bgProtectedRanges[i].Type == BG_PROTECTED_RANGE_INTEL_BOOT_GUARD_POST_IBB) { - if (!bgDxeCoreIndex.isValid()) { + else if (protectedRanges[i].Type == PROTECTED_RANGE_INTEL_BOOT_GUARD_POST_IBB) { + if (!dxeCore.isValid()) { msg(usprintf("%s: can't determine DXE volume offset, post-IBB protected range hash can't be checked", __FUNCTION__), index); } else { // Offset will be determined as the offset of root volume with first DXE core - UModelIndex dxeRootVolumeIndex = model->findLastParentOfType(bgDxeCoreIndex, Types::Volume); + UModelIndex dxeRootVolumeIndex = model->findLastParentOfType(dxeCore, Types::Volume); if (!dxeRootVolumeIndex.isValid()) { msg(usprintf("%s: can't determine DXE volume offset, post-IBB protected range hash can't be checked", __FUNCTION__), index); } else { - bgProtectedRanges[i].Offset = model->base(dxeRootVolumeIndex); - bgProtectedRanges[i].Size = (UINT32)(model->header(dxeRootVolumeIndex).size() + model->body(dxeRootVolumeIndex).size() + model->tail(dxeRootVolumeIndex).size()); - protectedParts = openedImage.mid(bgProtectedRanges[i].Offset, bgProtectedRanges[i].Size); + protectedRanges[i].Offset = model->base(dxeRootVolumeIndex); + protectedRanges[i].Size = (UINT32)(model->header(dxeRootVolumeIndex).size() + model->body(dxeRootVolumeIndex).size() + model->tail(dxeRootVolumeIndex).size()); + protectedParts = openedImage.mid(protectedRanges[i].Offset, protectedRanges[i].Size); - UByteArray digest(SHA256_DIGEST_SIZE, '\x00'); - sha256(protectedParts.constData(), protectedParts.size(), digest.data()); - - if (digest != bgProtectedRanges[i].Hash) { - msg(usprintf("%s: post-IBB protected range [%Xh:%Xh] hash mismatch, opened image may refuse to boot", __FUNCTION__, - bgProtectedRanges[i].Offset, bgProtectedRanges[i].Offset + bgProtectedRanges[i].Size), - model->findByBase(bgProtectedRanges[i].Offset)); + // Calculate the hash + UByteArray digest(SHA512_HASH_SIZE, '\x00'); + if (protectedRanges[i].AlgorithmId == TCG_HASH_ALGORITHM_ID_SHA1) { + sha1(protectedParts.constData(), protectedParts.size(), digest.data()); + digest = digest.left(SHA1_HASH_SIZE); + } + else if (protectedRanges[i].AlgorithmId == TCG_HASH_ALGORITHM_ID_SHA256) { + sha256(protectedParts.constData(), protectedParts.size(), digest.data()); + digest = digest.left(SHA256_HASH_SIZE); + } + else if (protectedRanges[i].AlgorithmId == TCG_HASH_ALGORITHM_ID_SHA384) { + sha384(protectedParts.constData(), protectedParts.size(), digest.data()); + digest = digest.left(SHA384_HASH_SIZE); + } + else if (protectedRanges[i].AlgorithmId == TCG_HASH_ALGORITHM_ID_SHA512) { + sha512(protectedParts.constData(), protectedParts.size(), digest.data()); + digest = digest.left(SHA512_HASH_SIZE); + } + else if (protectedRanges[i].AlgorithmId == TCG_HASH_ALGORITHM_ID_SM3) { + sm3(protectedParts.constData(), protectedParts.size(), digest.data()); + digest = digest.left(SM3_HASH_SIZE); + } + else { + msg(usprintf("%s: post-IBB protected range [%Xh:%Xh] uses unknown hash algorithm %04Xh", __FUNCTION__, + protectedRanges[i].Offset, protectedRanges[i].Offset + protectedRanges[i].Size, protectedRanges[i].AlgorithmId), + model->findByBase(protectedRanges[i].Offset)); } - markProtectedRangeRecursive(index, bgProtectedRanges[i]); + // Check the hash + if (digest != protectedRanges[i].Hash) { + msg(usprintf("%s: post-IBB protected range [%Xh:%Xh] hash mismatch, opened image may refuse to boot", __FUNCTION__, + protectedRanges[i].Offset, protectedRanges[i].Offset + protectedRanges[i].Size), + model->findByBase(protectedRanges[i].Offset)); + } + + markProtectedRangeRecursive(index, protectedRanges[i]); } } } - else if (bgProtectedRanges[i].Type == BG_PROTECTED_RANGE_VENDOR_HASH_AMI_NEW - && bgProtectedRanges[i].Size != 0 && bgProtectedRanges[i].Size != 0xFFFFFFFF - && bgProtectedRanges[i].Offset != 0 && bgProtectedRanges[i].Offset != 0xFFFFFFFF) { - - if ((UINT64)bgProtectedRanges[i].Offset >= addressDiff) { - bgProtectedRanges[i].Offset -= (UINT32)addressDiff; - protectedParts = openedImage.mid(bgProtectedRanges[i].Offset, bgProtectedRanges[i].Size); + else if (protectedRanges[i].Type == PROTECTED_RANGE_VENDOR_HASH_AMI_V2) { + if ((UINT64)protectedRanges[i].Offset >= addressDiff) { + protectedRanges[i].Offset -= (UINT32)addressDiff; + protectedParts = openedImage.mid(protectedRanges[i].Offset, protectedRanges[i].Size); - UByteArray digest(SHA256_DIGEST_SIZE, '\x00'); + UByteArray digest(SHA256_HASH_SIZE, '\x00'); sha256(protectedParts.constData(), protectedParts.size(), digest.data()); - if (digest != bgProtectedRanges[i].Hash) { + if (digest != protectedRanges[i].Hash) { msg(usprintf("%s: AMI protected range [%Xh:%Xh] hash mismatch, opened image may refuse to boot", __FUNCTION__, - bgProtectedRanges[i].Offset, bgProtectedRanges[i].Offset + bgProtectedRanges[i].Size), - model->findByBase(bgProtectedRanges[i].Offset)); + protectedRanges[i].Offset, protectedRanges[i].Offset + protectedRanges[i].Size), + model->findByBase(protectedRanges[i].Offset)); } - markProtectedRangeRecursive(index, bgProtectedRanges[i]); + markProtectedRangeRecursive(index, protectedRanges[i]); } else { - // TODO: Explore this. msg(usprintf("%s: suspicious AMI new BG protection offset", __FUNCTION__), index); } } - else if (bgProtectedRanges[i].Type == BG_PROTECTED_RANGE_VENDOR_HASH_PHOENIX - && bgProtectedRanges[i].Size != 0 && bgProtectedRanges[i].Size != 0xFFFFFFFF - && bgProtectedRanges[i].Offset != 0xFFFFFFFF) { - bgProtectedRanges[i].Offset += (UINT32)bgProtectedRegionsBase; - protectedParts = openedImage.mid(bgProtectedRanges[i].Offset, bgProtectedRanges[i].Size); + else if (protectedRanges[i].Type == PROTECTED_RANGE_VENDOR_HASH_PHOENIX + && protectedRanges[i].Size != 0 && protectedRanges[i].Size != 0xFFFFFFFF + && protectedRanges[i].Offset != 0xFFFFFFFF) { + protectedRanges[i].Offset += (UINT32)protectedRegionsBase; + protectedParts = openedImage.mid(protectedRanges[i].Offset, protectedRanges[i].Size); - UByteArray digest(SHA256_DIGEST_SIZE, '\x00'); + UByteArray digest(SHA256_HASH_SIZE, '\x00'); sha256(protectedParts.constData(), protectedParts.size(), digest.data()); - if (digest != bgProtectedRanges[i].Hash) { + if (digest != protectedRanges[i].Hash) { msg(usprintf("%s: Phoenix protected range [%Xh:%Xh] hash mismatch, opened image may refuse to boot", __FUNCTION__, - bgProtectedRanges[i].Offset, bgProtectedRanges[i].Offset + bgProtectedRanges[i].Size), - model->findByBase(bgProtectedRanges[i].Offset)); + protectedRanges[i].Offset, protectedRanges[i].Offset + protectedRanges[i].Size), + model->findByBase(protectedRanges[i].Offset)); } - markProtectedRangeRecursive(index, bgProtectedRanges[i]); + markProtectedRangeRecursive(index, protectedRanges[i]); } - else if (bgProtectedRanges[i].Type == BG_PROTECTED_RANGE_VENDOR_HASH_MICROSOFT - && bgProtectedRanges[i].Size != 0 && bgProtectedRanges[i].Size != 0xFFFFFFFF - && bgProtectedRanges[i].Offset != 0 && bgProtectedRanges[i].Offset != 0xFFFFFFFF) { - bgProtectedRanges[i].Offset -= (UINT32)addressDiff; - protectedParts = openedImage.mid(bgProtectedRanges[i].Offset, bgProtectedRanges[i].Size); + else if (protectedRanges[i].Type == PROTECTED_RANGE_VENDOR_HASH_MICROSOFT_PMDA + && protectedRanges[i].Size != 0 && protectedRanges[i].Size != 0xFFFFFFFF + && protectedRanges[i].Offset != 0 && protectedRanges[i].Offset != 0xFFFFFFFF) { + protectedRanges[i].Offset -= (UINT32)addressDiff; + protectedParts = openedImage.mid(protectedRanges[i].Offset, protectedRanges[i].Size); - UByteArray digest(SHA256_DIGEST_SIZE, '\x00'); - sha256(protectedParts.constData(), protectedParts.size(), digest.data()); - - if (digest != bgProtectedRanges[i].Hash) { - msg(usprintf("%s: Microsoft protected range [%Xh:%Xh] hash mismatch, opened image may refuse to boot", __FUNCTION__, - bgProtectedRanges[i].Offset, bgProtectedRanges[i].Offset + bgProtectedRanges[i].Size), - model->findByBase(bgProtectedRanges[i].Offset)); + // Calculate the hash + UByteArray digest(SHA512_HASH_SIZE, '\x00'); + if (protectedRanges[i].AlgorithmId == TCG_HASH_ALGORITHM_ID_SHA1) { + sha1(protectedParts.constData(), protectedParts.size(), digest.data()); + digest = digest.left(SHA1_HASH_SIZE); + } + else if (protectedRanges[i].AlgorithmId == TCG_HASH_ALGORITHM_ID_SHA256) { + sha256(protectedParts.constData(), protectedParts.size(), digest.data()); + digest = digest.left(SHA256_HASH_SIZE); + } + else if (protectedRanges[i].AlgorithmId == TCG_HASH_ALGORITHM_ID_SHA384) { + sha384(protectedParts.constData(), protectedParts.size(), digest.data()); + digest = digest.left(SHA384_HASH_SIZE); + } + else if (protectedRanges[i].AlgorithmId == TCG_HASH_ALGORITHM_ID_SHA512) { + sha512(protectedParts.constData(), protectedParts.size(), digest.data()); + digest = digest.left(SHA512_HASH_SIZE); + } + else if (protectedRanges[i].AlgorithmId == TCG_HASH_ALGORITHM_ID_SM3) { + sm3(protectedParts.constData(), protectedParts.size(), digest.data()); + digest = digest.left(SM3_HASH_SIZE); + } + else { + msg(usprintf("%s: Microsoft PMDA protected range [%Xh:%Xh] uses unknown hash algorithm %04Xh", __FUNCTION__, + protectedRanges[i].Offset, protectedRanges[i].Offset + protectedRanges[i].Size, protectedRanges[i].AlgorithmId), + model->findByBase(protectedRanges[i].Offset)); } - markProtectedRangeRecursive(index, bgProtectedRanges[i]); + // Check the hash + if (digest != protectedRanges[i].Hash) { + msg(usprintf("%s: Microsoft PMDA protected range [%Xh:%Xh] hash mismatch, opened image may refuse to boot", __FUNCTION__, + protectedRanges[i].Offset, protectedRanges[i].Offset + protectedRanges[i].Size), + model->findByBase(protectedRanges[i].Offset)); + } + + markProtectedRangeRecursive(index, protectedRanges[i]); } } return U_SUCCESS; } -USTATUS FfsParser::markProtectedRangeRecursive(const UModelIndex & index, const BG_PROTECTED_RANGE & range) +USTATUS FfsParser::markProtectedRangeRecursive(const UModelIndex & index, const PROTECTED_RANGE & range) { if (!index.isValid()) return U_SUCCESS; @@ -3532,7 +3622,7 @@ USTATUS FfsParser::markProtectedRangeRecursive(const UModelIndex & index, const if (std::min(currentOffset + currentSize, range.Offset + range.Size) > std::max(currentOffset, range.Offset)) { if (range.Offset <= currentOffset && currentOffset + currentSize <= range.Offset + range.Size) { // Mark as fully in range - if (range.Type == BG_PROTECTED_RANGE_INTEL_BOOT_GUARD_IBB) { + if (range.Type == PROTECTED_RANGE_INTEL_BOOT_GUARD_IBB) { model->setMarking(index, Qt::red); } else { @@ -3554,24 +3644,26 @@ USTATUS FfsParser::markProtectedRangeRecursive(const UModelIndex & index, const USTATUS FfsParser::parseVendorHashFile(const UByteArray & fileGuid, const UModelIndex & index) { - if (!index.isValid()) - return EFI_INVALID_PARAMETER; + // Check sanity + if (!index.isValid()) { + return U_INVALID_PARAMETER; + } - if (fileGuid == BG_VENDOR_HASH_FILE_GUID_PHOENIX) { + if (fileGuid == PROTECTED_RANGE_VENDOR_HASH_FILE_GUID_PHOENIX) { const UByteArray &body = model->body(index); UINT32 size = (UINT32)body.size(); // File too small to have even a signature - if (size < sizeof(BG_VENDOR_HASH_FILE_SIGNATURE_PHOENIX)) { + if (size < sizeof(UINT64)) { msg(usprintf("%s: unknown or corrupted Phoenix hash file found", __FUNCTION__), index); model->setText(index, UString("Phoenix hash file")); return U_INVALID_FILE; } - const BG_VENDOR_HASH_FILE_HEADER_PHOENIX* header = (const BG_VENDOR_HASH_FILE_HEADER_PHOENIX*)body.constData(); + const PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_PHOENIX* header = (const PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_PHOENIX*)body.constData(); if (header->Signature == BG_VENDOR_HASH_FILE_SIGNATURE_PHOENIX) { - if (size < sizeof(BG_VENDOR_HASH_FILE_HEADER_PHOENIX) || - size < sizeof(BG_VENDOR_HASH_FILE_HEADER_PHOENIX) + header->NumEntries * sizeof(BG_VENDOR_HASH_FILE_ENTRY)) { + if (size < sizeof(PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_PHOENIX) || + size < sizeof(PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_PHOENIX) + header->NumEntries * sizeof(PROTECTED_RANGE_VENDOR_HASH_FILE_ENTRY)) { msg(usprintf("%s: unknown or corrupted Phoenix hash file found", __FUNCTION__), index); model->setText(index, UString("Phoenix hash file")); return U_INVALID_FILE; @@ -3581,26 +3673,28 @@ USTATUS FfsParser::parseVendorHashFile(const UByteArray & fileGuid, const UModel bool protectedRangesFound = false; for (UINT32 i = 0; i < header->NumEntries; i++) { protectedRangesFound = true; - const BG_VENDOR_HASH_FILE_ENTRY* entry = (const BG_VENDOR_HASH_FILE_ENTRY*)(header + 1) + i; + const PROTECTED_RANGE_VENDOR_HASH_FILE_ENTRY* entry = (const PROTECTED_RANGE_VENDOR_HASH_FILE_ENTRY*)(header + 1) + i; - BG_PROTECTED_RANGE range = {}; - range.Offset = entry->Offset; + PROTECTED_RANGE range = {}; + range.Offset = entry->Base; range.Size = entry->Size; + range.AlgorithmId = TCG_HASH_ALGORITHM_ID_SHA256; range.Hash = UByteArray((const char*)entry->Hash, sizeof(entry->Hash)); - range.Type = BG_PROTECTED_RANGE_VENDOR_HASH_PHOENIX; - bgProtectedRanges.push_back(range); + range.Type = PROTECTED_RANGE_VENDOR_HASH_PHOENIX; + protectedRanges.push_back(range); } if (protectedRangesFound) { - securityInfo += usprintf("Phoenix hash file found at base %Xh\nProtected ranges:", model->base(index)); + securityInfo += usprintf("Phoenix hash file found at base %08Xh\nProtected ranges:", model->base(index)); for (UINT32 i = 0; i < header->NumEntries; i++) { - const BG_VENDOR_HASH_FILE_ENTRY* entry = (const BG_VENDOR_HASH_FILE_ENTRY*)(header + 1) + i; - securityInfo += usprintf("\nRelativeOffset: %08Xh Size: %Xh\nHash: ", entry->Offset, entry->Size); + const PROTECTED_RANGE_VENDOR_HASH_FILE_ENTRY* entry = (const PROTECTED_RANGE_VENDOR_HASH_FILE_ENTRY*)(header + 1) + i; + securityInfo += usprintf("RelativeOffset: %08Xh Size: %Xh\nHash: ", entry->Base, entry->Size); for (UINT8 j = 0; j < sizeof(entry->Hash); j++) { securityInfo += usprintf("%02X", entry->Hash[j]); } + securityInfo += "\n"; } - securityInfo += UString("\n------------------------------------------------------------------------\n\n"); + securityInfo += "\n"; } msg(usprintf("%s: Phoenix hash file found", __FUNCTION__), index); @@ -3612,54 +3706,67 @@ USTATUS FfsParser::parseVendorHashFile(const UByteArray & fileGuid, const UModel model->setText(index, UString("Phoenix hash file")); } } - else if (fileGuid == BG_VENDOR_HASH_FILE_GUID_AMI) { + else if (fileGuid == PROTECTED_RANGE_VENDOR_HASH_FILE_GUID_AMI) { UModelIndex fileIndex = model->parent(index); const UByteArray &body = model->body(index); UINT32 size = (UINT32)body.size(); if (size != (UINT32)body.count('\xFF')) { - if (size == sizeof(BG_VENDOR_HASH_FILE_HEADER_AMI_NEW)) { - bool protectedRangesFound = false; - UINT32 NumEntries = (UINT32)body.size() / sizeof(BG_VENDOR_HASH_FILE_ENTRY); - for (UINT32 i = 0; i < NumEntries; i++) { - protectedRangesFound = true; - const BG_VENDOR_HASH_FILE_ENTRY* entry = (const BG_VENDOR_HASH_FILE_ENTRY*)(body.constData()) + i; - BG_PROTECTED_RANGE range = {}; - range.Offset = entry->Offset; - range.Size = entry->Size; - range.Hash = UByteArray((const char*)entry->Hash, sizeof(entry->Hash)); - range.Type = BG_PROTECTED_RANGE_VENDOR_HASH_AMI_NEW; - bgProtectedRanges.push_back(range); + if (size == sizeof(PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V2)) { + const PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V2* entry = (const PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V2*)(body.constData()); + + securityInfo += usprintf("AMI hash file v2 found at base %08Xh\nProtected ranges:", model->base(fileIndex)); + securityInfo += usprintf("\nAddress: %08Xh, Size: %Xh\nHash (SHA256): ", entry->Hash0.Base, entry->Hash0.Size); + for (UINT8 j = 0; j < sizeof(entry->Hash0.Hash); j++) { + securityInfo += usprintf("%02X", entry->Hash0.Hash[j]); + } + securityInfo += usprintf("\nAddress: %08Xh, Size: %Xh\nHash (SHA256): ", entry->Hash1.Base, entry->Hash1.Size); + for (UINT8 j = 0; j < sizeof(entry->Hash1.Hash); j++) { + securityInfo += usprintf("%02X", entry->Hash1.Hash[j]); + } + securityInfo += "\n"; + + if (entry->Hash0.Base != 0 && entry->Hash0.Size != 0 + && entry->Hash0.Base != 0xFFFFFFFF && entry->Hash0.Size != 0xFFFFFFFF) { + PROTECTED_RANGE range = {}; + range.Offset = entry->Hash0.Base; + range.Size = entry->Hash0.Size; + range.AlgorithmId = TCG_HASH_ALGORITHM_ID_SHA256; + range.Hash = UByteArray((const char*)entry->Hash0.Hash, sizeof(entry->Hash0.Hash)); + range.Type = PROTECTED_RANGE_VENDOR_HASH_AMI_V2; + protectedRanges.push_back(range); } - if (protectedRangesFound) { - securityInfo += usprintf("New AMI hash file found at base %Xh\nProtected ranges:", model->base(fileIndex)); - for (UINT32 i = 0; i < NumEntries; i++) { - const BG_VENDOR_HASH_FILE_ENTRY* entry = (const BG_VENDOR_HASH_FILE_ENTRY*)(body.constData()) + i; - securityInfo += usprintf("\nAddress: %08Xh Size: %Xh\nHash: ", entry->Offset, entry->Size); - for (UINT8 j = 0; j < sizeof(entry->Hash); j++) { - securityInfo += usprintf("%02X", entry->Hash[j]); - } - } - securityInfo += UString("\n------------------------------------------------------------------------\n\n"); + if (entry->Hash1.Base != 0 && entry->Hash1.Size != 0 + && entry->Hash1.Base != 0xFFFFFFFF && entry->Hash1.Size != 0xFFFFFFFF) { + PROTECTED_RANGE range = {}; + range.Offset = entry->Hash1.Base; + range.Size = entry->Hash1.Size; + range.AlgorithmId = TCG_HASH_ALGORITHM_ID_SHA256; + range.Hash = UByteArray((const char*)entry->Hash1.Hash, sizeof(entry->Hash1.Hash)); + range.Type = PROTECTED_RANGE_VENDOR_HASH_AMI_V2; + protectedRanges.push_back(range); } - + msg(usprintf("%s: new AMI hash file found", __FUNCTION__), fileIndex); } - else if (size == sizeof(BG_VENDOR_HASH_FILE_HEADER_AMI_OLD)) { - securityInfo += usprintf("Old AMI hash file found at base %Xh\nProtected range:", model->base(fileIndex)); - const BG_VENDOR_HASH_FILE_HEADER_AMI_OLD* entry = (const BG_VENDOR_HASH_FILE_HEADER_AMI_OLD*)(body.constData()); - securityInfo += usprintf("\nSize: %Xh\nHash: ", entry->Size); + else if (size == sizeof(PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V1)) { + securityInfo += usprintf("AMI hash file v1 found at base %08Xh\nProtected range:\n", model->base(fileIndex)); + const PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V1* entry = (const PROTECTED_RANGE_VENDOR_HASH_FILE_HEADER_AMI_V1*)(body.constData()); + securityInfo += usprintf("Size: %Xh\nHash (SHA256): ", entry->Size); for (UINT8 i = 0; i < sizeof(entry->Hash); i++) { securityInfo += usprintf("%02X", entry->Hash[i]); } - securityInfo += UString("\n------------------------------------------------------------------------\n\n"); + securityInfo += "\n\n"; - BG_PROTECTED_RANGE range = {}; - range.Offset = 0; - range.Size = entry->Size; - range.Hash = UByteArray((const char*)entry->Hash, sizeof(entry->Hash)); - range.Type = BG_PROTECTED_RANGE_VENDOR_HASH_AMI_OLD; - bgProtectedRanges.push_back(range); + if (entry->Size != 0 && entry->Size != 0xFFFFFFFF) { + PROTECTED_RANGE range = {}; + range.Offset = 0; + range.Size = entry->Size; + range.AlgorithmId = TCG_HASH_ALGORITHM_ID_SHA256; + range.Hash = UByteArray((const char*)entry->Hash, sizeof(entry->Hash)); + range.Type = PROTECTED_RANGE_VENDOR_HASH_AMI_V1; + protectedRanges.push_back(range); + } msg(usprintf("%s: old AMI hash file found", __FUNCTION__), fileIndex); } @@ -3677,610 +3784,6 @@ USTATUS FfsParser::parseVendorHashFile(const UByteArray & fileGuid, const UModel return U_SUCCESS; } -#ifndef U_ENABLE_FIT_PARSING_SUPPORT -USTATUS FfsParser::parseFit(const UModelIndex & index) -{ - U_UNUSED_PARAMETER(index); - return U_SUCCESS; -} - -#else -USTATUS FfsParser::parseFit(const UModelIndex & index) -{ - // Check sanity - if (!index.isValid()) - return EFI_INVALID_PARAMETER; - - // Search for FIT - UModelIndex fitIndex; - UINT32 fitOffset; - findFitRecursive(index, fitIndex, fitOffset); - - // FIT not found - if (!fitIndex.isValid()) - return U_SUCCESS; - - // Explicitly set the item containing FIT as fixed - model->setFixed(fitIndex, true); - - // Special case of FIT header - UByteArray fitBody = model->body(fitIndex); - const FIT_ENTRY* fitHeader = (const FIT_ENTRY*)(fitBody.constData() + fitOffset); - - // Check FIT checksum, if present - UINT32 fitSize = fitHeader->Size * sizeof(FIT_ENTRY); - if (fitHeader->CsFlag) { - // Calculate FIT entry checksum - UByteArray tempFIT = model->body(fitIndex).mid(fitOffset, fitSize); - FIT_ENTRY* tempFitHeader = (FIT_ENTRY*)tempFIT.data(); - tempFitHeader->Checksum = 0; - UINT8 calculated = calculateChecksum8((const UINT8*)tempFitHeader, fitSize); - if (calculated != fitHeader->Checksum) { - msg(usprintf("%s: invalid FIT table checksum %02Xh, should be %02Xh", __FUNCTION__, fitHeader->Checksum, calculated), fitIndex); - } - } - - // Check fit header type - if (fitHeader->Type != FIT_TYPE_HEADER) { - msg(usprintf("%s: invalid FIT header type", __FUNCTION__), fitIndex); - return U_INVALID_FIT; - } - - // Add FIT header - std::vector currentStrings; - currentStrings.push_back(UString("_FIT_ ")); - currentStrings.push_back(usprintf("%08Xh", fitSize)); - currentStrings.push_back(usprintf("%04Xh", fitHeader->Version)); - currentStrings.push_back(usprintf("%02Xh", fitHeader->Checksum)); - currentStrings.push_back(fitEntryTypeToUString(fitHeader->Type)); - currentStrings.push_back(UString()); // Empty info for FIT header - fitTable.push_back(std::pair, UModelIndex>(currentStrings, fitIndex)); - - // Process all other entries - UModelIndex acmIndex; - UModelIndex kmIndex; - UModelIndex bpIndex; - for (UINT32 i = 1; i < fitHeader->Size; i++) { - currentStrings.clear(); - UString info; - UModelIndex itemIndex; - const FIT_ENTRY* currentEntry = fitHeader + i; - UINT32 currentEntrySize = currentEntry->Size; - - // Check sanity - if (currentEntry->Type == FIT_TYPE_HEADER) { - msg(usprintf("%s: second FIT header found, the table is damaged", __FUNCTION__), fitIndex); - return U_INVALID_FIT; - } - - // Special case of version 0 entries - if (currentEntry->Version == 0) { - const FIT_ENTRY_VERSION_0_CONFIG_POLICY* policy = (const FIT_ENTRY_VERSION_0_CONFIG_POLICY*)currentEntry; - info += usprintf("Index: %04Xh BitPosition: %02Xh AccessWidth: %02Xh DataRegAddr: %04Xh IndexRegAddr: %04Xh", - policy->Index, - policy->BitPosition, - policy->AccessWidth, - policy->DataRegisterAddress, - policy->IndexRegisterAddress); - } - else if (currentEntry->Address > addressDiff && currentEntry->Address < 0xFFFFFFFFUL) { // Only elements in the image need to be parsed - UINT32 currentEntryBase = (UINT32)(currentEntry->Address - addressDiff); - itemIndex = model->findByBase(currentEntryBase); - if (itemIndex.isValid()) { - USTATUS status = U_INVALID_FIT; - UByteArray item = model->header(itemIndex) + model->body(itemIndex) + model->tail(itemIndex); - UINT32 localOffset = currentEntryBase - model->base(itemIndex); - - switch (currentEntry->Type) { - case FIT_TYPE_MICROCODE: - status = parseFitEntryMicrocode(item, localOffset, itemIndex, info, currentEntrySize); - break; - - case FIT_TYPE_BIOS_AC_MODULE: - status = parseFitEntryAcm(item, localOffset, itemIndex, info, currentEntrySize); - acmIndex = itemIndex; - break; - - case FIT_TYPE_AC_KEY_MANIFEST: - status = parseFitEntryBootGuardKeyManifest(item, localOffset, itemIndex, info, currentEntrySize); - kmIndex = itemIndex; - break; - - case FIT_TYPE_AC_BOOT_POLICY: - status = parseFitEntryBootGuardBootPolicy(item, localOffset, itemIndex, info, currentEntrySize); - bpIndex = itemIndex; - break; - - default: - // Do nothing - status = U_SUCCESS; - break; - } - - if (status != U_SUCCESS) - itemIndex = UModelIndex(); - } - else { - msg(usprintf("%s: FIT entry #%d not found in the image", __FUNCTION__, i), fitIndex); - } - } - - if (itemIndex.isValid()) { - // Explicitly set the item referenced by FIT as fixed - // TODO: lift this restriction after FIT builder is ready - model->setFixed(itemIndex, true); - } - - // Add entry to fitTable - currentStrings.push_back(usprintf("%016" PRIX64 "h", currentEntry->Address)); - currentStrings.push_back(usprintf("%08Xh", currentEntrySize)); - currentStrings.push_back(usprintf("%04Xh", currentEntry->Version)); - currentStrings.push_back(usprintf("%02Xh", currentEntry->Checksum)); - currentStrings.push_back(fitEntryTypeToUString(currentEntry->Type)); - currentStrings.push_back(info); - fitTable.push_back(std::pair, UModelIndex>(currentStrings, itemIndex)); - } - - // Perform validation of BootGuard stuff - if (bgAcmFound) { - if (!bgKeyManifestFound) { - msg(usprintf("%s: ACM found, but KeyManifest is not", __FUNCTION__), acmIndex); - } - else if (!bgBootPolicyFound) { - msg(usprintf("%s: ACM and KeyManifest found, BootPolicy is not", __FUNCTION__), kmIndex); - } - else { - // Check key hashes - if (!bgKmHash.isEmpty() && bgBpHash.isEmpty() && bgKmHash != bgBpHash) { - msg(usprintf("%s: BootPolicy key hash stored in KeyManifest differs from the hash of public key stored in BootPolicy", __FUNCTION__), bpIndex); - return U_SUCCESS; - } - } - } - - return U_SUCCESS; -} - -void FfsParser::findFitRecursive(const UModelIndex & index, UModelIndex & found, UINT32 & fitOffset) -{ - // Sanity check - if (!index.isValid()) { - return; - } - - // Process child items - for (int i = 0; i < model->rowCount(index); i++) { - findFitRecursive(index.model()->index(i, 0, index), found, fitOffset); - - if (found.isValid()) - return; - } - - // Check for all FIT signatures in item's body - UByteArray lastVtfBody = model->body(lastVtf); - UINT32 storedFitAddress = *(const UINT32*)(lastVtfBody.constData() + lastVtfBody.size() - FIT_POINTER_OFFSET); - for (INT32 offset = (INT32)model->body(index).indexOf(FIT_SIGNATURE); - offset >= 0; - offset = (INT32)model->body(index).indexOf(FIT_SIGNATURE, offset + 1)) { - // FIT candidate found, calculate its physical address - UINT32 fitAddress = (UINT32)(model->base(index) + (UINT32)addressDiff + model->header(index).size() + (UINT32)offset); - - // Check FIT address to be stored in the last VTF - if (fitAddress == storedFitAddress) { - found = index; - fitOffset = offset; - msg(usprintf("%s: real FIT table found at physical address %08Xh", __FUNCTION__, fitAddress), found); - break; - } - else if (model->rowCount(index) == 0) // Show messages only to leaf items - msg(usprintf("%s: FIT table candidate found, but not referenced from the last VTF", __FUNCTION__), index); - } -} - -USTATUS FfsParser::parseFitEntryMicrocode(const UByteArray & microcode, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize) -{ - U_UNUSED_PARAMETER(parent); - if ((UINT32)microcode.size() - localOffset < sizeof(INTEL_MICROCODE_HEADER)) { - return U_INVALID_MICROCODE; - } - - const INTEL_MICROCODE_HEADER* ucodeHeader = (const INTEL_MICROCODE_HEADER*)(microcode.constData() + localOffset); - if (!microcodeHeaderValid(ucodeHeader)) { - return U_INVALID_MICROCODE; - } - - if ((UINT32)microcode.size() - localOffset < ucodeHeader->TotalSize) { - return U_INVALID_MICROCODE; - } - - // Valid microcode found - info = usprintf("CpuSignature: %08Xh, Revision: %08Xh, Date: %02X.%02X.%04X", - ucodeHeader->ProcessorSignature, - ucodeHeader->UpdateRevision, - ucodeHeader->DateDay, - ucodeHeader->DateMonth, - ucodeHeader->DateYear); - realSize = ucodeHeader->TotalSize; - return U_SUCCESS; -} - -USTATUS FfsParser::parseFitEntryAcm(const UByteArray & acm, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize) -{ - if ((UINT32)acm.size() < localOffset + sizeof(INTEL_ACM_HEADER)) { - return U_INVALID_ACM; - } - - const INTEL_ACM_HEADER* header = (const INTEL_ACM_HEADER*)(acm.constData() + localOffset); - if (header->ModuleType != INTEL_ACM_MODULE_TYPE || header->ModuleVendor != INTEL_ACM_MODULE_VENDOR) { - return U_INVALID_ACM; - } - - UINT32 acmSize = header->ModuleSize * sizeof(UINT32); - if ((UINT32)acm.size() < localOffset + acmSize) { - return U_INVALID_ACM; - } - - // Valid ACM found - info = usprintf("LocalOffset: %08Xh, EntryPoint: %08Xh, ACM SVN: %04Xh, Date: %02X.%02X.%04X", - localOffset, - header->EntryPoint, - header->AcmSvn, - header->DateDay, - header->DateMonth, - header->DateYear - ); - realSize = acmSize; - - // Add ACM header info - UString acmInfo; - acmInfo += usprintf(" found at base %Xh\n" - "ModuleType: %04Xh ModuleSubtype: %04Xh HeaderLength: %08lXh\n" - "HeaderVersion: %08Xh ChipsetId: %04Xh Flags: %04Xh\n" - "ModuleVendor: %04Xh Date: %02X.%02X.%04X ModuleSize: %08lXh\n" - "EntryPoint: %08Xh AcmSvn: %04Xh Unknown1: %08Xh\n" - "Unknown2: %08Xh GdtBase: %08Xh GdtMax: %08Xh\n" - "SegSel: %08Xh KeySize: %08lXh Unknown3: %08lXh", - model->base(parent) + localOffset, - header->ModuleType, - header->ModuleSubtype, - header->ModuleSize * sizeof(UINT32), - header->HeaderVersion, - header->ChipsetId, - header->Flags, - header->ModuleVendor, - header->DateDay, header->DateMonth, header->DateYear, - header->ModuleSize * sizeof(UINT32), - header->EntryPoint, - header->AcmSvn, - header->Unknown1, - header->Unknown2, - header->GdtBase, - header->GdtMax, - header->SegmentSel, - header->KeySize * sizeof(UINT32), - header->Unknown4 * sizeof(UINT32) - ); - // Add PubKey - acmInfo += usprintf("\n\nACM RSA Public Key (Exponent: %Xh):", header->RsaPubExp); - for (UINT16 i = 0; i < sizeof(header->RsaPubKey); i++) { - if (i % 32 == 0) - acmInfo += UString("\n"); - acmInfo += usprintf("%02X", header->RsaPubKey[i]); - } - // Add RsaSig - acmInfo += UString("\n\nACM RSA Signature:"); - for (UINT16 i = 0; i < sizeof(header->RsaSig); i++) { - if (i % 32 == 0) - acmInfo += UString("\n"); - acmInfo += usprintf("%02X", header->RsaSig[i]); - } - acmInfo += UString("\n------------------------------------------------------------------------\n\n"); - - if(header->ModuleSubtype == INTEL_ACM_MODULE_SUBTYPE_TXT_ACM) - securityInfo += "TXT ACM" + acmInfo; - else if(header->ModuleSubtype == INTEL_ACM_MODULE_SUBTYPE_S_ACM) - securityInfo += "S-ACM" + acmInfo; - else if (header->ModuleSubtype == INTEL_ACM_MODULE_SUBTYPE_BOOTGUARD) - securityInfo += "BootGuard ACM" + acmInfo; - else - securityInfo += "Intel ACM" + acmInfo; - - bgAcmFound = true; - return U_SUCCESS; -} - -USTATUS FfsParser::parseFitEntryBootGuardKeyManifest(const UByteArray & keyManifest, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize) -{ - U_UNUSED_PARAMETER(realSize); - if ((UINT32)keyManifest.size() < localOffset + sizeof(BG_KEY_MANIFEST)) { - return U_INVALID_BG_KEY_MANIFEST; - } - - const BG_KEY_MANIFEST* header = (const BG_KEY_MANIFEST*)(keyManifest.constData() + localOffset); - if (header->Tag != BG_KEY_MANIFEST_TAG) { - return U_INVALID_BG_KEY_MANIFEST; - } - - // Valid KM found - info = usprintf("LocalOffset: %08Xh, KM Version: %02Xh, KM SVN: %02Xh, KM ID: %02Xh", - localOffset, - header->KmVersion, - header->KmSvn, - header->KmId - ); - - // Add KM header info - securityInfo += usprintf("Intel BootGuard Key manifest found at base %Xh\n" - "Tag: __KEYM__ Version: %02Xh KmVersion: %02Xh KmSvn: %02Xh KmId: %02Xh", - model->base(parent) + localOffset, - header->Version, - header->KmVersion, - header->KmSvn, - header->KmId - ); - - // Add hash of Key Manifest PubKey, this hash will be written to FPFs - UINT8 hash[SHA256_DIGEST_SIZE]; - sha256(&header->KeyManifestSignature.PubKey.Modulus, sizeof(header->KeyManifestSignature.PubKey.Modulus), hash); - securityInfo += UString("\n\nKey Manifest RSA Public Key Hash:\n"); - for (UINT8 i = 0; i < sizeof(hash); i++) { - securityInfo += usprintf("%02X", hash[i]); - } - - // Add BpKeyHash - securityInfo += UString("\n\nBoot Policy RSA Public Key Hash:\n"); - for (UINT8 i = 0; i < sizeof(header->BpKeyHash.HashBuffer); i++) { - securityInfo += usprintf("%02X", header->BpKeyHash.HashBuffer[i]); - } - bgKmHash = UByteArray((const char*)header->BpKeyHash.HashBuffer, sizeof(header->BpKeyHash.HashBuffer)); - - // Add Key Manifest PubKey - securityInfo += usprintf("\n\nKey Manifest RSA Public Key (Exponent: %Xh):", - header->KeyManifestSignature.PubKey.Exponent); - for (UINT16 i = 0; i < sizeof(header->KeyManifestSignature.PubKey.Modulus); i++) { - if (i % 32 == 0) - securityInfo += UString("\n"); - securityInfo += usprintf("%02X", header->KeyManifestSignature.PubKey.Modulus[i]); - } - // Add Key Manifest Signature - securityInfo += UString("\n\nKey Manifest RSA Signature:"); - for (UINT16 i = 0; i < sizeof(header->KeyManifestSignature.Signature.Signature); i++) { - if (i % 32 == 0) - securityInfo += UString("\n"); - securityInfo += usprintf("%02X", header->KeyManifestSignature.Signature.Signature[i]); - } - securityInfo += UString("\n------------------------------------------------------------------------\n\n"); - bgKeyManifestFound = true; - return U_SUCCESS; -} - -USTATUS FfsParser::findNextBootGuardBootPolicyElement(const UByteArray & bootPolicy, const UINT32 elementOffset, UINT32 & nextElementOffset, UINT32 & nextElementSize) -{ - UINT32 dataSize = (UINT32)bootPolicy.size(); - if (dataSize < sizeof(UINT64)) { - return U_ELEMENTS_NOT_FOUND; - } - - UINT32 offset = elementOffset; - for (; offset < dataSize - sizeof(UINT64); offset++) { - const UINT64* currentPos = (const UINT64*)(bootPolicy.constData() + offset); - if (*currentPos == BG_BOOT_POLICY_MANIFEST_IBB_ELEMENT_TAG && offset + sizeof(BG_IBB_ELEMENT) < dataSize) { - const BG_IBB_ELEMENT* header = (const BG_IBB_ELEMENT*)currentPos; - // Check that all segments are present - if (offset + sizeof(BG_IBB_ELEMENT) + sizeof(BG_IBB_SEGMENT_ELEMENT) * header->IbbSegCount < dataSize) { - nextElementOffset = offset; - nextElementSize = sizeof(BG_IBB_ELEMENT) + sizeof(BG_IBB_SEGMENT_ELEMENT) * header->IbbSegCount; - return U_SUCCESS; - } - } - else if (*currentPos == BG_BOOT_POLICY_MANIFEST_PLATFORM_MANUFACTURER_ELEMENT_TAG && offset + sizeof(BG_PLATFORM_MANUFACTURER_ELEMENT) < dataSize) { - const BG_PLATFORM_MANUFACTURER_ELEMENT* header = (const BG_PLATFORM_MANUFACTURER_ELEMENT*)currentPos; - // Check that data is present - if (offset + sizeof(BG_PLATFORM_MANUFACTURER_ELEMENT) + header->DataSize < dataSize) { - nextElementOffset = offset; - nextElementSize = sizeof(BG_PLATFORM_MANUFACTURER_ELEMENT) + header->DataSize; - return U_SUCCESS; - } - } - else if (*currentPos == BG_BOOT_POLICY_MANIFEST_SIGNATURE_ELEMENT_TAG && offset + sizeof(BG_BOOT_POLICY_MANIFEST_SIGNATURE_ELEMENT) < dataSize) { - nextElementOffset = offset; - nextElementSize = sizeof(BG_BOOT_POLICY_MANIFEST_SIGNATURE_ELEMENT); - return U_SUCCESS; - } - } - - return U_ELEMENTS_NOT_FOUND; -} - -USTATUS FfsParser::parseFitEntryBootGuardBootPolicy(const UByteArray & bootPolicy, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize) -{ - U_UNUSED_PARAMETER(realSize); - if ((UINT32)bootPolicy.size() < localOffset + sizeof(BG_BOOT_POLICY_MANIFEST_HEADER)) { - return U_INVALID_BG_BOOT_POLICY; - } - - const BG_BOOT_POLICY_MANIFEST_HEADER* header = (const BG_BOOT_POLICY_MANIFEST_HEADER*)(bootPolicy.constData() + localOffset); - if (header->Tag != BG_BOOT_POLICY_MANIFEST_HEADER_TAG) { - return U_INVALID_BG_BOOT_POLICY; - } - - UINT32 bmSize = sizeof(BG_BOOT_POLICY_MANIFEST_HEADER); - if ((UINT32)bootPolicy.size() < localOffset + bmSize) { - return U_INVALID_BG_BOOT_POLICY; - } - - // Valid BPM found - info = usprintf("LocalOffset: %08Xh, BP SVN: %02Xh, ACM SVN: %02Xh", - localOffset, - header->BPSVN, - header->ACMSVN - ); - - // Add BP header info - securityInfo += usprintf( - "Intel BootGuard Boot Policy Manifest found at base %Xh\n" - "Tag: __ACBP__ Version: %02Xh HeaderVersion: %02Xh\n" - "PMBPMVersion: %02Xh PBSVN: %02Xh ACMSVN: %02Xh NEMDataStack: %04Xh\n", - model->base(parent) + localOffset, - header->Version, - header->HeaderVersion, - header->PMBPMVersion, - header->BPSVN, - header->ACMSVN, - header->NEMDataSize - ); - - // Iterate over elements to get them all - UINT32 elementOffset = 0; - UINT32 elementSize = 0; - USTATUS status = findNextBootGuardBootPolicyElement(bootPolicy, localOffset + sizeof(BG_BOOT_POLICY_MANIFEST_HEADER), elementOffset, elementSize); - while (status == U_SUCCESS) { - const UINT64* currentPos = (const UINT64*)(bootPolicy.constData() + elementOffset); - if (*currentPos == BG_BOOT_POLICY_MANIFEST_IBB_ELEMENT_TAG) { - const BG_IBB_ELEMENT* elementHeader = (const BG_IBB_ELEMENT*)currentPos; - // Valid IBB element found - securityInfo += usprintf( - "\nInitial Boot Block Element found at base %Xh\n" - "Tag: __IBBS__ Version: %02Xh Unknown: %02Xh\n" - "Flags: %08Xh IbbMchBar: %08llXh VtdBar: %08llXh\n" - "PmrlBase: %08Xh PmrlLimit: %08Xh EntryPoint: %08Xh", - model->base(parent) + localOffset + elementOffset, - elementHeader->Version, - elementHeader->Unknown, - elementHeader->Flags, - (unsigned long long)elementHeader->IbbMchBar, - (unsigned long long)elementHeader->VtdBar, - elementHeader->PmrlBase, - elementHeader->PmrlLimit, - elementHeader->EntryPoint - ); - - // Add PostIbbHash - securityInfo += UString("\n\nPost IBB Hash:\n"); - for (UINT8 i = 0; i < sizeof(elementHeader->IbbHash.HashBuffer); i++) { - securityInfo += usprintf("%02X", elementHeader->IbbHash.HashBuffer[i]); - } - - // Check for non-empry PostIbbHash - UByteArray postIbbHash((const char*)elementHeader->IbbHash.HashBuffer, sizeof(elementHeader->IbbHash.HashBuffer)); - if (postIbbHash.count('\x00') != postIbbHash.size() && postIbbHash.count('\xFF') != postIbbHash.size()) { - BG_PROTECTED_RANGE range = {}; - range.Type = BG_PROTECTED_RANGE_INTEL_BOOT_GUARD_POST_IBB; - range.Hash = postIbbHash; - bgProtectedRanges.push_back(range); - } - - // Add Digest - bgBpDigest = UByteArray((const char*)elementHeader->Digest.HashBuffer, sizeof(elementHeader->Digest.HashBuffer)); - securityInfo += UString("\n\nIBB Digest:\n"); - for (UINT8 i = 0; i < (UINT8)bgBpDigest.size(); i++) { - securityInfo += usprintf("%02X", (UINT8)bgBpDigest.at(i)); - } - - // Add all IBB segments - securityInfo += UString("\n\nIBB Segments:\n"); - const BG_IBB_SEGMENT_ELEMENT* segments = (const BG_IBB_SEGMENT_ELEMENT*)(elementHeader + 1); - for (UINT8 i = 0; i < elementHeader->IbbSegCount; i++) { - securityInfo += usprintf("Flags: %04Xh Address: %08Xh Size: %08Xh\n", - segments[i].Flags, segments[i].Base, segments[i].Size); - if (segments[i].Flags == BG_IBB_SEGMENT_FLAG_IBB) { - BG_PROTECTED_RANGE range = {}; - range.Offset = segments[i].Base; - range.Size = segments[i].Size; - range.Type = BG_PROTECTED_RANGE_INTEL_BOOT_GUARD_IBB; - bgProtectedRanges.push_back(range); - } - } - } - else if (*currentPos == BG_BOOT_POLICY_MANIFEST_PLATFORM_MANUFACTURER_ELEMENT_TAG) { - const BG_PLATFORM_MANUFACTURER_ELEMENT* elementHeader = (const BG_PLATFORM_MANUFACTURER_ELEMENT*)currentPos; - securityInfo += usprintf( - "\nPlatform Manufacturer Data Element found at base %Xh\n" - "Tag: __PMDA__ Version: %02Xh DataSize: %02Xh", - model->base(parent) + localOffset + elementOffset, - elementHeader->Version, - elementHeader->DataSize - ); - // Check for Microsoft PMDA hash data - const BG_MICROSOFT_PMDA_HEADER* pmdaHeader = (const BG_MICROSOFT_PMDA_HEADER*)(elementHeader + 1); - if (pmdaHeader->Version == BG_MICROSOFT_PMDA_VERSION - && elementHeader->DataSize == sizeof(BG_MICROSOFT_PMDA_HEADER) + sizeof(BG_MICROSOFT_PMDA_ENTRY)*pmdaHeader->NumEntries) { - // Add entries - securityInfo += UString("\nMicrosoft PMDA-based protected ranges:\n"); - const BG_MICROSOFT_PMDA_ENTRY* entries = (const BG_MICROSOFT_PMDA_ENTRY*)(pmdaHeader + 1); - for (UINT32 i = 0; i < pmdaHeader->NumEntries; i++) { - - securityInfo += usprintf("Address: %08Xh Size: %08Xh\n", entries[i].Address, entries[i].Size); - securityInfo += UString("Hash: "); - for (UINT8 j = 0; j < sizeof(entries[i].Hash); j++) { - securityInfo += usprintf("%02X", entries[i].Hash[j]); - } - securityInfo += UString("\n"); - - BG_PROTECTED_RANGE range = {}; - range.Offset = entries[i].Address; - range.Size = entries[i].Size; - range.Hash = UByteArray((const char*)entries[i].Hash, sizeof(entries[i].Hash)); - range.Type = BG_PROTECTED_RANGE_VENDOR_HASH_MICROSOFT; - bgProtectedRanges.push_back(range); - } - } - else { - // Add raw data - const UINT8* data = (const UINT8*)(elementHeader + 1); - for (UINT16 i = 0; i < elementHeader->DataSize; i++) { - if (i % 32 == 0) - securityInfo += UString("\n"); - securityInfo += usprintf("%02X", data[i]); - } - securityInfo += UString("\n"); - } - } - else if (*currentPos == BG_BOOT_POLICY_MANIFEST_SIGNATURE_ELEMENT_TAG) { - const BG_BOOT_POLICY_MANIFEST_SIGNATURE_ELEMENT* elementHeader = (const BG_BOOT_POLICY_MANIFEST_SIGNATURE_ELEMENT*)currentPos; - securityInfo += usprintf( - "\nBoot Policy Signature Element found at base %Xh\n" - "Tag: __PMSG__ Version: %02Xh", - model->base(parent) + localOffset + elementOffset, - elementHeader->Version - ); - - // Add PubKey - securityInfo += usprintf("\n\nBoot Policy RSA Public Key (Exponent: %Xh):", elementHeader->KeySignature.PubKey.Exponent); - for (UINT16 i = 0; i < sizeof(elementHeader->KeySignature.PubKey.Modulus); i++) { - if (i % 32 == 0) - securityInfo += UString("\n"); - securityInfo += usprintf("%02X", elementHeader->KeySignature.PubKey.Modulus[i]); - } - - // Calculate and add PubKey hash - UINT8 hash[SHA256_DIGEST_SIZE]; - sha256(&elementHeader->KeySignature.PubKey.Modulus, sizeof(elementHeader->KeySignature.PubKey.Modulus), hash); - securityInfo += UString("\n\nBoot Policy RSA Public Key Hash:"); - for (UINT8 i = 0; i < sizeof(hash); i++) { - if (i % 32 == 0) - securityInfo += UString("\n"); - securityInfo += usprintf("%02X", hash[i]); - } - bgBpHash = UByteArray((const char*)hash, sizeof(hash)); - - // Add Signature - securityInfo += UString("\n\nBoot Policy RSA Signature:"); - for (UINT16 i = 0; i < sizeof(elementHeader->KeySignature.Signature.Signature); i++) { - if (i % 32 == 0) - securityInfo += UString("\n"); - securityInfo += usprintf("%02X", elementHeader->KeySignature.Signature.Signature[i]); - } - } - status = findNextBootGuardBootPolicyElement(bootPolicy, elementOffset + elementSize, elementOffset, elementSize); - } - - securityInfo += UString("\n------------------------------------------------------------------------\n\n"); - bgBootPolicyFound = true; - return U_SUCCESS; -} -#endif - USTATUS FfsParser::parseMicrocodeVolumeBody(const UModelIndex & index) { const UINT32 headerSize = (UINT32)model->header(index).size(); @@ -4645,7 +4148,7 @@ make_partition_table_consistent: parseCpdRegion(partition, 0, partitionIndex, cpdIndex); } - // TODO: make this generic again + // There needs to be a more generic way to do it, but it is fine for now if (partitions[i].ptEntry.Type > BPDT_ENTRY_TYPE_TBT && partitions[i].ptEntry.Type != BPDT_ENTRY_TYPE_USB_PHY && partitions[i].ptEntry.Type != BPDT_ENTRY_TYPE_PCHC @@ -4986,7 +4489,7 @@ make_partition_table_consistent: + (partitions[i].ptEntry.Offset.HuffmanCompressed ? "Yes" : "No"); // Calculate SHA256 hash over the metadata and add it to its info - UByteArray hash(SHA256_DIGEST_SIZE, '\x00'); + UByteArray hash(SHA256_HASH_SIZE, '\x00'); sha256(partition.constData(), partition.size(), hash.data()); info += UString("\nMetadata hash: ") + UString(hash.toHex().constData()); @@ -5003,7 +4506,7 @@ make_partition_table_consistent: + (partitions[i].ptEntry.Offset.HuffmanCompressed ? "Yes" : "No"); // Calculate SHA256 hash over the code and add it to its info - UByteArray hash(SHA256_DIGEST_SIZE, '\x00'); + UByteArray hash(SHA256_HASH_SIZE, '\x00'); sha256(partition.constData(), partition.size(), hash.data()); info += UString("\nHash: ") + UString(hash.toHex().constData()); @@ -5040,7 +4543,8 @@ USTATUS FfsParser::parseCpdExtensionsArea(const UModelIndex & index) UINT32 offset = 0; while (offset < (UINT32)body.size()) { const CPD_EXTENTION_HEADER* extHeader = (const CPD_EXTENTION_HEADER*) (body.constData() + offset); - if (extHeader->Length > 0 && extHeader->Length <= ((UINT32)body.size() - offset)) { + if (extHeader->Length > 0 + && extHeader->Length <= ((UINT32)body.size() - offset)) { UByteArray partition = body.mid(offset, extHeader->Length); UString name = cpdExtensionTypeToUstring(extHeader->Type); @@ -5146,7 +4650,7 @@ USTATUS FfsParser::parseCpdExtensionsArea(const UModelIndex & index) extIndex = model->addItem(offset, Types::CpdExtension, 0, name, UString(), info, UByteArray(), partition, UByteArray(), Fixed, index); } - // TODO: make this generic again + // There needs to be a more generic way to do it, but it is fine for now if (extHeader->Type > CPD_EXT_TYPE_TBT_METADATA && extHeader->Type != CPD_EXT_TYPE_GMF_CERTIFICATE && extHeader->Type != CPD_EXT_TYPE_GMF_BODY @@ -5229,9 +4733,9 @@ void FfsParser::outputInfo(void) { // Get security info UString secInfo = getSecurityInfo(); if (!secInfo.isEmpty()) { - std::cout << "------------------------------------------------------------------------" << std::endl; + std::cout << "---------------------------------------------------------------------------" << std::endl; std::cout << "Security Info" << std::endl; - std::cout << "------------------------------------------------------------------------" << std::endl; + std::cout << "---------------------------------------------------------------------------" << std::endl; std::cout << (const char *)secInfo.toLocal8Bit() << std::endl; } } diff --git a/common/ffsparser.h b/common/ffsparser.h index 6395853..884b898 100644 --- a/common/ffsparser.h +++ b/common/ffsparser.h @@ -19,23 +19,28 @@ WITHWARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. #include "ustring.h" #include "ubytearray.h" #include "treemodel.h" -#include "bootguard.h" -#include "fit.h" +#include "intel_microcode.h" +#include "fitparser.h" -typedef struct BG_PROTECTED_RANGE_ { +typedef struct PROTECTED_RANGE_ { UINT32 Offset; UINT32 Size; + UINT16 AlgorithmId; UINT8 Type; + UINT8 : 8; UByteArray Hash; -} BG_PROTECTED_RANGE; +} PROTECTED_RANGE; -#define BG_PROTECTED_RANGE_INTEL_BOOT_GUARD_IBB 0x01 -#define BG_PROTECTED_RANGE_INTEL_BOOT_GUARD_POST_IBB 0x02 -#define BG_PROTECTED_RANGE_VENDOR_HASH_PHOENIX 0x03 -#define BG_PROTECTED_RANGE_VENDOR_HASH_AMI_OLD 0x04 -#define BG_PROTECTED_RANGE_VENDOR_HASH_AMI_NEW 0x05 -#define BG_PROTECTED_RANGE_VENDOR_HASH_MICROSOFT 0x06 +#define PROTECTED_RANGE_INTEL_BOOT_GUARD_IBB 0x01 +#define PROTECTED_RANGE_INTEL_BOOT_GUARD_POST_IBB 0x02 +#define PROTECTED_RANGE_INTEL_BOOT_GUARD_OBB 0x03 +#define PROTECTED_RANGE_VENDOR_HASH_PHOENIX 0x04 +#define PROTECTED_RANGE_VENDOR_HASH_AMI_V1 0x05 +#define PROTECTED_RANGE_VENDOR_HASH_AMI_V2 0x06 +#define PROTECTED_RANGE_VENDOR_HASH_AMI_V3 0x07 +#define PROTECTED_RANGE_VENDOR_HASH_MICROSOFT_PMDA 0x08 +class FitParser; class NvramParser; class MeParser; @@ -55,10 +60,10 @@ public: USTATUS parse(const UByteArray &buffer); // Obtain parsed FIT table - std::vector, UModelIndex> > getFitTable() const { return fitTable; } + std::vector, UModelIndex> > getFitTable() const; // Obtain Security Info - UString getSecurityInfo() const { return securityInfo; } + UString getSecurityInfo() const; // Obtain offset/address difference UINT64 getAddressDiff() { return addressDiff; } @@ -73,6 +78,7 @@ private: messagesVector.push_back(std::pair(message, index)); }; + FitParser* fitParser; NvramParser* nvramParser; MeParser* meParser; @@ -80,18 +86,12 @@ private: UModelIndex lastVtf; UINT32 imageBase; UINT64 addressDiff; - std::vector, UModelIndex> > fitTable; UString securityInfo; - bool bgAcmFound; - bool bgKeyManifestFound; - bool bgBootPolicyFound; - UByteArray bgKmHash; - UByteArray bgBpHash; - UByteArray bgBpDigest; - std::vector bgProtectedRanges; - UINT64 bgProtectedRegionsBase; - UModelIndex bgDxeCoreIndex; + + std::vector protectedRanges; + UINT64 protectedRegionsBase; + UModelIndex dxeCore; // First pass USTATUS performFirstPass(const UByteArray & imageFile, UModelIndex & index); @@ -149,27 +149,20 @@ private: USTATUS parseIntelMicrocodeHeader(const UByteArray & store, const UINT32 localOffset, const UModelIndex & parent, UModelIndex & index); BOOLEAN microcodeHeaderValid(const INTEL_MICROCODE_HEADER* ucodeHeader); + USTATUS parseVendorHashFile(const UByteArray & fileGuid, const UModelIndex & index); + // Second pass USTATUS performSecondPass(const UModelIndex & index); USTATUS addInfoRecursive(const UModelIndex & index); USTATUS checkTeImageBase(const UModelIndex & index); + USTATUS checkProtectedRanges(const UModelIndex & index); - USTATUS markProtectedRangeRecursive(const UModelIndex & index, const BG_PROTECTED_RANGE & range); + USTATUS markProtectedRangeRecursive(const UModelIndex & index, const PROTECTED_RANGE & range); USTATUS parseResetVectorData(); - USTATUS parseFit(const UModelIndex & index); - USTATUS parseVendorHashFile(const UByteArray & fileGuid, const UModelIndex & index); - - + #ifdef U_ENABLE_FIT_PARSING_SUPPORT - void findFitRecursive(const UModelIndex & index, UModelIndex & found, UINT32 & fitOffset); - - // FIT entries - USTATUS parseFitEntryMicrocode(const UByteArray & microcode, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize); - USTATUS parseFitEntryAcm(const UByteArray & acm, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize); - USTATUS parseFitEntryBootGuardKeyManifest(const UByteArray & keyManifest, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize); - USTATUS parseFitEntryBootGuardBootPolicy(const UByteArray & bootPolicy, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize); - USTATUS findNextBootGuardBootPolicyElement(const UByteArray & bootPolicy, const UINT32 elementOffset, UINT32 & nextElementOffset, UINT32 & nextElementSize); + friend class FitParser; // Make FFS parsing routines accessible to FitParser #endif #ifdef U_ENABLE_NVRAM_PARSING_SUPPORT diff --git a/common/ffsreport.cpp b/common/ffsreport.cpp index 63c4128..0954185 100644 --- a/common/ffsreport.cpp +++ b/common/ffsreport.cpp @@ -14,7 +14,6 @@ #include "ffsreport.h" #include "ffs.h" #include "utility.h" -#include "uinttypes.h" std::vector FfsReport::generate() { diff --git a/common/fitparser.cpp b/common/fitparser.cpp new file mode 100644 index 0000000..1bdf0d7 --- /dev/null +++ b/common/fitparser.cpp @@ -0,0 +1,1116 @@ +/* fitparser.cpp + + Copyright (c) 2022, Nikolaj Schlej. All rights reserved. + This program and the accompanying materials + are licensed and made available under the terms and conditions of the BSD License + which accompanies this distribution. The full text of the license may be found at + http://opensource.org/licenses/bsd-license.php + + THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. + + */ +#include "fitparser.h" + +#ifdef U_ENABLE_FIT_PARSING_SUPPORT + +#include "intel_fit.h" +#include "ffs.h" +#include "parsingdata.h" +#include "types.h" +#include "utility.h" +#include "digest/sha2.h" + +#include +#include "kaitai/kaitaistream.h" +#include "generated/intel_acbp_v1.h" +#include "generated/intel_acbp_v2.h" +#include "generated/intel_keym_v1.h" +#include "generated/intel_keym_v2.h" +#include "generated/intel_acm.h" + +USTATUS FitParser::parseFit(const UModelIndex & index) +{ + // Reset parser state + fitTable.clear(); + securityInfo = ""; + bgAcmFound = false; + bgKeyManifestFound = false; + bgBootPolicyFound = false; + bgKmHash = UByteArray(); + bgBpHashSha256 = UByteArray(); + bgBpHashSha384 = UByteArray(); + + // Check sanity + if (!index.isValid()) { + return U_INVALID_PARAMETER; + } + + // Search for FIT + UModelIndex fitIndex; + UINT32 fitOffset; + findFitRecursive(index, fitIndex, fitOffset); + + // FIT not found + if (!fitIndex.isValid()) { + // Nothing to parse further + return U_SUCCESS; + } + // Explicitly set the item containing FIT as fixed + model->setFixed(fitIndex, true); + + // Special case of FIT header + UByteArray fitBody = model->body(fitIndex); + // This is safe, as we checked the size in findFitRecursive already + const INTEL_FIT_ENTRY* fitHeader = (const INTEL_FIT_ENTRY*)(fitBody.constData() + fitOffset); + + // Sanity check + UINT32 fitSize = fitHeader->Size * sizeof(INTEL_FIT_ENTRY); + if ((UINT32)fitBody.size() - fitOffset < fitSize) { + msg(usprintf("%s: not enough space to contain the whole FIT table", __FUNCTION__), fitIndex); + return U_INVALID_FIT; + } + + // Check FIT checksum, if present + if (fitHeader->ChecksumValid) { + // Calculate FIT entry checksum + UByteArray tempFIT = model->body(fitIndex).mid(fitOffset, fitSize); + INTEL_FIT_ENTRY* tempFitHeader = (INTEL_FIT_ENTRY*)tempFIT.data(); + tempFitHeader->Checksum = 0; + UINT8 calculated = calculateChecksum8((const UINT8*)tempFitHeader, fitSize); + if (calculated != fitHeader->Checksum) { + msg(usprintf("%s: invalid FIT table checksum %02Xh, should be %02Xh", __FUNCTION__, fitHeader->Checksum, calculated), fitIndex); + } + } + + // Check fit header type + if (fitHeader->Type != INTEL_FIT_TYPE_HEADER) { + msg(usprintf("%s: invalid FIT header type", __FUNCTION__), fitIndex); + return U_INVALID_FIT; + } + + // Add FIT header + std::vector currentStrings; + currentStrings.push_back(UString("_FIT_ ")); + currentStrings.push_back(usprintf("%08Xh", fitSize)); + currentStrings.push_back(usprintf("%04Xh", fitHeader->Version)); + currentStrings.push_back(usprintf("%02Xh", fitHeader->Checksum)); + currentStrings.push_back(fitEntryTypeToUString(fitHeader->Type)); + currentStrings.push_back(UString()); // Empty info for FIT header + fitTable.push_back(std::pair, UModelIndex>(currentStrings, fitIndex)); + + // Process all other entries + UModelIndex acmIndex; + UModelIndex kmIndex; + UModelIndex bpIndex; + for (UINT32 i = 1; i < fitHeader->Size; i++) { + currentStrings.clear(); + UString info; + UModelIndex itemIndex; + const INTEL_FIT_ENTRY* currentEntry = fitHeader + i; + UINT32 currentEntrySize = currentEntry->Size; + + // Check sanity + if (currentEntry->Type == INTEL_FIT_TYPE_HEADER) { + msg(usprintf("%s: second FIT header found, the table is damaged", __FUNCTION__), fitIndex); + return U_INVALID_FIT; + } + + // Special case of version 0 entries for TXT and TPM policies + if ((currentEntry->Type == INTEL_FIT_TYPE_TXT_POLICY || currentEntry->Type == INTEL_FIT_TYPE_TPM_POLICY) + && currentEntry->Version == 0) { + const INTEL_FIT_INDEX_IO_ADDRESS* policy = (const INTEL_FIT_INDEX_IO_ADDRESS*)currentEntry; + info += usprintf("Index: %04Xh, BitPosition: %02Xh, AccessWidth: %02Xh, DataRegAddr: %04Xh, IndexRegAddr: %04Xh", + policy->Index, + policy->BitPosition, + policy->AccessWidthInBytes, + policy->DataRegisterAddress, + policy->IndexRegisterAddress); + } + else if (currentEntry->Address > ffsParser->addressDiff && currentEntry->Address < 0xFFFFFFFFUL) { // Only elements in the image need to be parsed + UINT32 currentEntryBase = (UINT32)(currentEntry->Address - ffsParser->addressDiff); + itemIndex = model->findByBase(currentEntryBase); + if (itemIndex.isValid()) { + UByteArray item = model->header(itemIndex) + model->body(itemIndex) + model->tail(itemIndex); + UINT32 localOffset = currentEntryBase - model->base(itemIndex); + + switch (currentEntry->Type) { + case INTEL_FIT_TYPE_MICROCODE: + (void)parseFitEntryMicrocode(item, localOffset, itemIndex, info, currentEntrySize); + break; + + case INTEL_FIT_TYPE_STARTUP_AC_MODULE: + (void)parseFitEntryAcm(item, localOffset, itemIndex, info, currentEntrySize); + acmIndex = itemIndex; + break; + + case INTEL_FIT_TYPE_BOOT_GUARD_KEY_MANIFEST: + (void)parseFitEntryBootGuardKeyManifest(item, localOffset, itemIndex, info, currentEntrySize); + kmIndex = itemIndex; + break; + + case INTEL_FIT_TYPE_BOOT_GUARD_BOOT_POLICY: + (void)parseFitEntryBootGuardBootPolicy(item, localOffset, itemIndex, info, currentEntrySize); + bpIndex = itemIndex; + break; + + default: + // Do nothing + break; + } + } + else { + msg(usprintf("%s: FIT entry #%u not found in the image", __FUNCTION__, i), fitIndex); + } + } + + // Explicitly set the item referenced by FIT as fixed + if (itemIndex.isValid()) { + model->setFixed(itemIndex, true); + } + + // Add entry to fitTable + currentStrings.push_back(usprintf("%016" PRIX64 "h", currentEntry->Address)); + currentStrings.push_back(usprintf("%08Xh", currentEntrySize)); + currentStrings.push_back(usprintf("%04Xh", currentEntry->Version)); + currentStrings.push_back(usprintf("%02Xh", currentEntry->Checksum)); + currentStrings.push_back(fitEntryTypeToUString(currentEntry->Type)); + currentStrings.push_back(info); + fitTable.push_back(std::pair, UModelIndex>(currentStrings, itemIndex)); + } + + // Perform validation of BootGuard components + if (bgAcmFound) { + if (!bgKeyManifestFound) { + msg(usprintf("%s: startup ACM found, but KeyManifest is not", __FUNCTION__), acmIndex); + } + else if (!bgBootPolicyFound) { + msg(usprintf("%s: startup ACM and Key Manifest found, Boot Policy is not", __FUNCTION__), kmIndex); + } + else { + // Check key hashes + if (!bgKmHash.isEmpty() + && !(bgKmHash == bgBpHashSha256 || bgKmHash == bgBpHashSha384)) { + msg(usprintf("%s: Boot Policy key hash stored in Key Manifest differs from the hash of the public key stored in Boot Policy", __FUNCTION__), bpIndex); + return U_SUCCESS; + } + } + } + + return U_SUCCESS; +} + +void FitParser::findFitRecursive(const UModelIndex & index, UModelIndex & found, UINT32 & fitOffset) +{ + // Sanity check + if (!index.isValid()) { + return; + } + + // Process child items + for (int i = 0; i < model->rowCount(index); i++) { + findFitRecursive(index.model()->index(i, 0, index), found, fitOffset); + + if (found.isValid()) { + // Found it, no need to process further + return; + } + } + + // Check for all FIT signatures in item body + UByteArray lastVtfBody = model->body(ffsParser->lastVtf); + UINT64 fitSignatureValue = INTEL_FIT_SIGNATURE; + UByteArray fitSignature((const char*)&fitSignatureValue, sizeof(fitSignatureValue)); + UINT32 storedFitAddress = *(const UINT32*)(lastVtfBody.constData() + lastVtfBody.size() - INTEL_FIT_POINTER_OFFSET); + for (INT32 offset = (INT32)model->body(index).indexOf(fitSignature); + offset >= 0; + offset = (INT32)model->body(index).indexOf(fitSignature, offset + 1)) { + // FIT candidate found, calculate its physical address + UINT32 fitAddress = (UINT32)(model->base(index) + (UINT32)ffsParser->addressDiff + model->header(index).size() + (UINT32)offset); + + // Check FIT address to be stored in the last VTF + if (fitAddress == storedFitAddress) { + // Valid FIT table must have at least two entries + if ((UINT32)model->body(index).size() < offset + 2*sizeof(INTEL_FIT_ENTRY)) { + msg(usprintf("%s: FIT table candidate found, too small to contain real FIT", __FUNCTION__), index); + } + else { + // Real FIT found + found = index; + fitOffset = offset; + msg(usprintf("%s: real FIT table found at physical address %08Xh", __FUNCTION__, fitAddress), found); + break; + } + } + else if (model->rowCount(index) == 0) { // Show messages only to leaf items + msg(usprintf("%s: FIT table candidate found, but not referenced from the last VTF", __FUNCTION__), index); + } + } +} + +USTATUS FitParser::parseFitEntryMicrocode(const UByteArray & microcode, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize) +{ + U_UNUSED_PARAMETER(parent); + if ((UINT32)microcode.size() - localOffset < sizeof(INTEL_MICROCODE_HEADER)) { + return U_INVALID_MICROCODE; + } + + const INTEL_MICROCODE_HEADER* ucodeHeader = (const INTEL_MICROCODE_HEADER*)(microcode.constData() + localOffset); + if (!ffsParser->microcodeHeaderValid(ucodeHeader)) { + return U_INVALID_MICROCODE; + } + + if ((UINT32)microcode.size() - localOffset < ucodeHeader->TotalSize) { + return U_INVALID_MICROCODE; + } + + // Valid microcode found + info = usprintf("CpuSignature: %08Xh, Revision: %08Xh, Date: %02X.%02X.%04X", + ucodeHeader->ProcessorSignature, + ucodeHeader->UpdateRevision, + ucodeHeader->DateDay, + ucodeHeader->DateMonth, + ucodeHeader->DateYear); + realSize = ucodeHeader->TotalSize; + + return U_SUCCESS; +} + +USTATUS FitParser::parseFitEntryAcm(const UByteArray & acm, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize) +{ + // Parse Intel ACM using Kaitai-generated parser + // TODO: need to do something with the copy here, probably implement a custom wrapper + std::istringstream is(std::string(acm.constData(), acm.size())); + is.seekg(localOffset, is.beg); + kaitai::kstream ks(&is); + try { + intel_acm_t parsed(&ks); + intel_acm_t::header_t* header = parsed.header(); + + realSize = header->module_size(); + + // Check header version to be of a known value + if (header->header_version() != intel_acm_t::KNOWN_HEADER_VERSION_V0_0 + && header->header_version() != intel_acm_t::KNOWN_HEADER_VERSION_V3_0) { + msg(usprintf("%s: Intel ACM with unknown header version %08Xh found", __FUNCTION__, header->header_version()), parent); + } + + // Valid ACM found + info = usprintf("LocalOffset: %08Xh, EntryPoint: %08Xh, ACM SVN: %04Xh, Date: %02X.%02X.%04X", + localOffset, + header->entry_point(), + header->acm_svn(), + header->date_day(), + header->date_month(), + header->date_year()); + + // Populate ACM info + UString acmInfo; + if (header->module_subtype() == intel_acm_t::MODULE_SUBTYPE_TXT) { + acmInfo = "TXT ACM "; + } + else if(header->module_subtype() == intel_acm_t::MODULE_SUBTYPE_STARTUP) { + acmInfo = "Startup ACM "; + } + else if (header->module_subtype() == intel_acm_t::MODULE_SUBTYPE_BOOT_GUARD) { + acmInfo = "BootGuard ACM "; + } + else { + acmInfo = usprintf("Unknown ACM (%04Xh)", header->module_subtype()); + msg(usprintf("%s: Intel ACM with unknown subtype %04Xh found", __FUNCTION__, header->module_subtype()), parent); + } + + acmInfo += usprintf("found at base %Xh\n" + "ModuleType: %04Xh\n" + "ModuleSubtype: %04Xh\n" + "HeaderSize: %08Xh\n" + "HeaderVersion: %08Xh\n" + "ChipsetId: %04Xh\n" + "Flags: %04Xh\n" + "ModuleVendor: %04Xh\n" + "Date: %02X.%02X.%04X\n" + "ModuleSize: %08Xh\n" + "AcmSvn: %04Xh\n" + "SeSvn: %04Xh\n" + "CodeControlFlags: %08Xh\n" + "ErrorEntryPoint: %08Xh\n" + "GdtMax: %08Xh\n" + "GdtBase: %08Xh\n" + "SegmentSel: %08Xh\n" + "EntryPoint: %08Xh\n" + "KeySize: %08Xh\n" + "ScratchSpaceSize: %08Xh\n", + model->base(parent) + localOffset, + header->module_type(), + header->module_subtype(), + header->header_size() * (UINT32)sizeof(UINT32), + header->header_version(), + header->chipset_id(), + header->flags(), + header->module_vendor(), + header->date_day(), header->date_month(), header->date_year(), + header->module_size() * (UINT32)sizeof(UINT32), + header->acm_svn(), + header->se_svn(), + header->code_control_flags(), + header->error_entry_point(), + header->gdt_max(), + header->gdt_base(), + header->segment_sel(), + header->entry_point(), + header->key_size() * (UINT32)sizeof(UINT32), + header->scratch_space_size() * (UINT32)sizeof(UINT32)); + + // Add RsaPublicKey + if (header->_is_null_rsa_exponent() == false) { + acmInfo += usprintf("ACM RSA Public Key (Exponent: %Xh):", header->rsa_exponent()); + } + else { + acmInfo += usprintf("ACM RSA Public Key (Exponent: %Xh):", INTEL_ACM_HARDCODED_RSA_EXPONENT); + } + for (UINT32 i = 0; i < header->rsa_public_key().size(); i++) { + if (i % 32 == 0) acmInfo += "\n"; + acmInfo += usprintf("%02X", (UINT8)header->rsa_public_key().at(i)); + } + acmInfo += "\n"; + + // Add RsaSignature + acmInfo += UString("ACM RSA Signature:"); + for (UINT32 i = 0; i < header->rsa_signature().size(); i++) { + if (i % 32 == 0) acmInfo +="\n"; + acmInfo += usprintf("%02X", (UINT8)header->rsa_signature().at(i)); + } + acmInfo += "\n"; + + securityInfo += acmInfo + "\n"; + bgAcmFound = true; + return U_SUCCESS; + } + catch (...) { + msg(usprintf("%s: unable to parse ACM", __FUNCTION__), parent); + return U_INVALID_ACM; + } +} + +USTATUS FitParser::parseFitEntryBootGuardKeyManifest(const UByteArray & keyManifest, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize) +{ + U_UNUSED_PARAMETER(realSize); + + // TODO: need to do something with the copy here, probably implement a custom wrapper + std::istringstream is(std::string(keyManifest.constData(), keyManifest.size())); + + // v1 + try { + is.seekg(localOffset, is.beg); + kaitai::kstream ks(&is); + intel_keym_v1_t parsed(&ks); + + // Valid KM found + info = usprintf("LocalOffset: %08Xh, Version: %02Xh, KM Version: %02Xh, KM SVN: %02Xh", + localOffset, + parsed.version(), + parsed.km_version(), + parsed.km_svn()); + + // Populate KM info + UString kmInfo + = usprintf("Intel BootGuard Key manifest found at base %Xh\n" + "Tag: '__KEYM__'\n" + "Version: %02Xh\n" + "KmVersion: %02Xh\n" + "KmSvn: %02Xh\n" + "KmId: %02Xh\n", + model->base(parent) + localOffset, + parsed.version(), + parsed.km_version(), + parsed.km_svn(), + parsed.km_id()); + + // Add KM hash + kmInfo += UString("KM Hash (") + hashTypeToUString(parsed.km_hash()->hash_algorithm_id()) + "): "; + for (UINT16 j = 0; j < parsed.km_hash()->len_hash(); j++) { + kmInfo += usprintf("%02X", (UINT8) parsed.km_hash()->hash().data()[j]); + } + kmInfo += "\n"; + + // Add Key Signature + const intel_keym_v1_t::key_signature_t* key_signature = parsed.key_signature(); + kmInfo += usprintf("Key Manifest Key Signature:\n" + "Version: %02Xh\n" + "KeyId: %04Xh\n" + "SigScheme: %04Xh\n", + key_signature->version(), + key_signature->key_id(), + key_signature->sig_scheme()); + + // Add PubKey + kmInfo += usprintf("Key Manifest Public Key (Exponent: %Xh): ", key_signature->public_key()->exponent()); + for (UINT16 i = 0; i < (UINT16)key_signature->public_key()->modulus().length(); i++) { + if (i % 32 == 0) kmInfo += UString("\n"); + kmInfo += usprintf("%02X", (UINT8)key_signature->public_key()->modulus().at(i)); + } + kmInfo += "\n"; + + // Add Signature + kmInfo += UString("Key Manifest Signature: "); + for (UINT16 i = 0; i < (UINT16)key_signature->signature()->signature().length(); i++) { + if (i % 32 == 0) kmInfo += UString("\n"); + kmInfo += usprintf("%02X", (UINT8)key_signature->signature()->signature().at(i)); + } + kmInfo += "\n"; + + securityInfo += kmInfo + "\n"; + bgKeyManifestFound = true; + return U_SUCCESS; + } + catch (...) { + // Do nothing here, will try parsing as v2 next + } + + // v2 + try { + is.seekg(localOffset, is.beg); + kaitai::kstream ks(&is); + intel_keym_v2_t parsed(&ks); + intel_keym_v2_t::header_t* header = parsed.header(); + + // Valid KM found + info = usprintf("LocalOffset: %08Xh, Version: %02Xh, KM Version: %02Xh, KM SVN: %02Xh", + localOffset, + header->version(), + parsed.km_version(), + parsed.km_svn()); + + // Populate KM info + UString kmInfo + = usprintf("Intel BootGuard Key manifest found at base %Xh\n" + "Tag: '__KEYM__'\n" + "Version: %02Xh\n" + "KmVersion: %02Xh\n" + "KmSvn: %02Xh\n" + "KmId: %02Xh\n" + "KeySignatureOffset: %04Xh\n" + "FPFHashAlgorithmId: %04Xh\n" + "HashCount: %04Xh\n", + model->base(parent) + localOffset, + header->version(), + parsed.km_version(), + parsed.km_svn(), + parsed.km_id(), + parsed.key_signature_offset(), + parsed.fpf_hash_algorithm_id(), + parsed.num_km_hashes()); + + // Add KM hashes + if (parsed.num_km_hashes() == 0) { + kmInfo += UString("KM Hashes: N/A\n"); + msg(usprintf("%s: Key Manifest without KM hashes", __FUNCTION__), parent); + } + else { + kmInfo += UString("KM Hashes:\n"); + for (UINT16 i = 0; i < parsed.num_km_hashes(); i++) { + intel_keym_v2_t::km_hash_t* current_km_hash = parsed.km_hashes()->at(i); + + // Add KM hash + kmInfo += usprintf("UsageFlags: %016" PRIX64 "h, ", current_km_hash->usage_flags()) + hashTypeToUString(current_km_hash->hash_algorithm_id()) + ": "; + for (UINT16 j = 0; j < current_km_hash->len_hash(); j++) { + kmInfo += usprintf("%02X", (UINT8)current_km_hash->hash().data()[j]); + } + kmInfo += "\n"; + + if (current_km_hash->usage_flags() == intel_keym_v2_t::KM_USAGE_FLAGS_BOOT_POLICY_MANIFEST) { + bgKmHash = UByteArray((const char*)current_km_hash->hash().data(), current_km_hash->hash().size()); + } + } + } + + // Add Key Signature + const intel_keym_v2_t::key_signature_t* key_signature = parsed.key_signature(); + kmInfo += usprintf("Key Manifest Key Signature:\n" + "Version: %02Xh\n" + "KeyId: %04Xh\n" + "SigScheme: %04Xh\n", + key_signature->version(), + key_signature->key_id(), + key_signature->sig_scheme()); + + // Add PubKey + kmInfo += usprintf("Key Manifest Public Key (Exponent: %Xh): ", key_signature->public_key()->exponent()); + for (UINT16 i = 0; i < (UINT16)key_signature->public_key()->modulus().length(); i++) { + if (i % 32 == 0) kmInfo += UString("\n"); + kmInfo += usprintf("%02X", (UINT8)key_signature->public_key()->modulus().at(i)); + } + kmInfo += "\n"; + + // Add Signature + kmInfo += UString("Key Manifest Signature: "); + for (UINT16 i = 0; i < (UINT16)key_signature->signature()->signature().length(); i++) { + if (i % 32 == 0) kmInfo += UString("\n"); + kmInfo += usprintf("%02X", (UINT8)key_signature->signature()->signature().at(i)); + } + kmInfo += "\n"; + + securityInfo += kmInfo + "\n"; + bgKeyManifestFound = true; + return U_SUCCESS; + } + catch (...) { + msg(usprintf("%s: unable to parse Key Manifest", __FUNCTION__), parent); + return U_INVALID_BOOT_GUARD_KEY_MANIFEST; + } +} + +USTATUS FitParser::parseFitEntryBootGuardBootPolicy(const UByteArray & bootPolicy, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize) +{ + U_UNUSED_PARAMETER(realSize); + + // TODO: need to do something with the copy here, probably implement a custom wrapper + std::istringstream is(std::string(bootPolicy.constData(), bootPolicy.size())); + + // v1 + try { + is.seekg(localOffset, is.beg); + kaitai::kstream ks(&is); + intel_acbp_v1_t parsed(&ks); + + // Valid BPM found + info = usprintf("LocalOffset: %08Xh, Version: %02Xh, BP SVN: %02Xh, ACM SVN: %02Xh", + localOffset, + parsed.version(), + parsed.bp_svn(), + parsed.acm_svn()); + + UString bpInfo + = usprintf("Intel BootGuard Boot Policy Manifest found at base %Xh\n" + "StructureId: '__ACBP__'\n" + "Version: %02Xh\n" + "BPMRevision: %02Xh\n" + "BPSVN: %02Xh\n" + "ACMSVN: %02Xh\n" + "NEMDataSize: %04Xh\n", + model->base(parent) + localOffset, + parsed.version(), + parsed.bpm_revision(), + parsed.bp_svn(), + parsed.acm_svn(), + parsed.nem_data_size()); + + bpInfo += UString("Boot Policy Elements:\n"); + const std::vector* elements = parsed.elements(); + for (intel_acbp_v1_t::acbp_element_t* element : *elements) { + const intel_acbp_v1_t::common_header_t* element_header = element->header(); + + UINT64 structure_id = element_header->structure_id(); + const char* structure_id_bytes = (const char*)&structure_id; + + bpInfo += usprintf("StructureId: '%c%c%c%c%c%c%c%c'\n" + "Version: %02Xh\n", + structure_id_bytes[0], + structure_id_bytes[1], + structure_id_bytes[2], + structure_id_bytes[3], + structure_id_bytes[4], + structure_id_bytes[5], + structure_id_bytes[6], + structure_id_bytes[7], + element_header->version()); + + // IBBS + if (element->_is_null_ibbs_body() == false) { + const intel_acbp_v1_t::ibbs_body_t* ibbs_body = element->ibbs_body(); + + // Valid IBBS element found + bpInfo += usprintf("Flags: %08Xh\n" + "MchBar: %016" PRIX64 "h\n" + "VtdBar: %016" PRIX64 "h\n" + "DmaProtectionBase0: %08Xh\n" + "DmaProtectionLimit0: %08Xh\n" + "DmaProtectionBase1: %016" PRIX64 "h\n" + "DmaProtectionLimit1: %016" PRIX64 "h\n" + "IbbEntryPoint: %08Xh\n" + "IbbSegmentsCount: %02Xh\n", + ibbs_body->flags(), + ibbs_body->mch_bar(), + ibbs_body->vtd_bar(), + ibbs_body->dma_protection_base0(), + ibbs_body->dma_protection_limit0(), + ibbs_body->dma_protection_base1(), + ibbs_body->dma_protection_limit1(), + ibbs_body->ibb_entry_point(), + ibbs_body->num_ibb_segments()); + + // Check for non-empty PostIbbHash + if (ibbs_body->post_ibb_hash()->len_hash() == 0) { + bpInfo += UString("PostIBB Hash: N/A\n"); + } + else { + // Add postIbbHash protected range + UByteArray postIbbHash(ibbs_body->post_ibb_hash()->hash().data(), ibbs_body->post_ibb_hash()->len_hash()); + if (postIbbHash.count('\x00') != postIbbHash.size() + && postIbbHash.count('\xFF') != postIbbHash.size()) { + PROTECTED_RANGE range = {}; + range.Type = PROTECTED_RANGE_INTEL_BOOT_GUARD_POST_IBB; + range.AlgorithmId = ibbs_body->post_ibb_hash()->hash_algorithm_id(); + range.Hash = postIbbHash; + ffsParser->protectedRanges.push_back(range); + } + + // Add PostIbbHash + bpInfo += UString("PostIBB Hash (") + hashTypeToUString(ibbs_body->post_ibb_hash()->hash_algorithm_id()) + "): "; + for (UINT16 i = 0; i < ibbs_body->post_ibb_hash()->len_hash(); i++) { + bpInfo += usprintf("%02X", (UINT8)ibbs_body->post_ibb_hash()->hash().data()[i]); + } + bpInfo += "\n"; + } + + // Add IbbHash + bpInfo += UString("IBB Hash (") + hashTypeToUString(ibbs_body->ibb_hash()->hash_algorithm_id()) + "): "; + for (UINT16 j = 0; j < ibbs_body->ibb_hash()->len_hash(); j++) { + bpInfo += usprintf("%02X", (UINT8)ibbs_body->ibb_hash()->hash().data()[j]); + } + bpInfo += "\n"; + + // Check for non-empty IbbSegments + if (ibbs_body->num_ibb_segments() == 0) { + bpInfo += UString("IBB Segments: N/A\n"); + msg(usprintf("%s: Boot Policy without IBB segments", __FUNCTION__), parent); + } + else { + bpInfo += UString("IBB Segments:\n"); + for (UINT8 i = 0; i < ibbs_body->num_ibb_segments(); i++) { + const intel_acbp_v1_t::ibb_segment_t* current_segment = ibbs_body->ibb_segments()->at(i); + + bpInfo += usprintf("Flags: %04Xh, Address: %08Xh, Size: %08Xh\n", + current_segment->flags(), + current_segment->base(), + current_segment->size()); + + if (current_segment->flags() == intel_acbp_v1_t::IBB_SEGMENT_TYPE_IBB) { + PROTECTED_RANGE range = {}; + range.Offset = current_segment->base(); + range.Size = current_segment->size(); + range.AlgorithmId = TCG_HASH_ALGORITHM_ID_SHA256; + range.Type = PROTECTED_RANGE_INTEL_BOOT_GUARD_IBB; + ffsParser->protectedRanges.push_back(range); + } + } + } + } + // PMDA + else if (element->_is_null_pmda_body() == false) { + intel_acbp_v1_t::pmda_body_t* pmda_body = element->pmda_body(); + + // Valid Microsoft PMDA element found + bpInfo += usprintf("TotalSize: %04Xh\n" + "Version: %08Xh\n" + "NumEntries: %08Xh\n", + pmda_body->total_size(), + pmda_body->version(), + pmda_body->num_entries()); + if (pmda_body->num_entries() == 0) { + bpInfo += UString("PMDA Entries: N/A\n"); + } + else { + bpInfo += UString("PMDA Entries:\n"); + // v1 entries + if (pmda_body->_is_null_entries_v1() == false) { + for (UINT32 i = 0; i < pmda_body->num_entries(); i++) { + const intel_acbp_v1_t::pmda_entry_v1_t* current_element = pmda_body->entries_v1()->at(i); + + // Add element + bpInfo += usprintf("Address: %08Xh, Size: %08Xh\n", + current_element->base(), + current_element->size()); + + // Add hash + bpInfo += "SHA256: "; + for (UINT16 j = 0; j < (UINT16)current_element->hash().size(); j++) { + bpInfo += usprintf("%02X", (UINT8)current_element->hash().data()[j]); + } + bpInfo += "\n"; + + // Add protected range + PROTECTED_RANGE range = {}; + range.Offset = current_element->base(); + range.Size = current_element->size(); + range.Type = PROTECTED_RANGE_VENDOR_HASH_MICROSOFT_PMDA; + range.AlgorithmId = TCG_HASH_ALGORITHM_ID_SHA256; + range.Hash = UByteArray(current_element->hash().data(), current_element->hash().size()); + ffsParser->protectedRanges.push_back(range); + } + } + // v2 entries + else if (pmda_body->_is_null_entries_v2() == false) { + for (UINT32 i = 0; i < pmda_body->num_entries(); i++) { + const intel_acbp_v1_t::pmda_entry_v2_t* current_element = pmda_body->entries_v2()->at(i); + + // Add element + bpInfo += usprintf("Address: %08Xh, Size: %08Xh\n", + current_element->base(), + current_element->size()); + + // Add hash + bpInfo += hashTypeToUString(current_element->hash()->hash_algorithm_id()) + ": "; + for (UINT16 j = 0; j < (UINT16)current_element->hash()->hash().size(); j++) { + bpInfo += usprintf("%02X", (UINT8)current_element->hash()->hash().data()[j]); + } + bpInfo += "\n"; + + // Add protected range + PROTECTED_RANGE range = {}; + range.Offset = current_element->base(); + range.Size = current_element->size(); + range.Type = PROTECTED_RANGE_VENDOR_HASH_MICROSOFT_PMDA; + range.AlgorithmId = current_element->hash()->hash_algorithm_id(); + range.Hash = UByteArray(current_element->hash()->hash().data(), current_element->hash()->hash().size()); + ffsParser->protectedRanges.push_back(range); + } + } + } + } + // PMSG + else if (element->_is_null_pmsg_body() == false) { + const intel_acbp_v1_t::pmsg_body_t* key_signature = element->pmsg_body(); + bpInfo += usprintf("Boot Policy Key Signature:\n" + "Version: %02Xh\n" + "KeyId: %04Xh\n" + "SigScheme: %04Xh\n", + key_signature->version(), + key_signature->key_id(), + key_signature->sig_scheme()); + + // Add PubKey + bpInfo += usprintf("Boot Policy Public Key (Exponent: %Xh): ", key_signature->public_key()->exponent()); + for (UINT16 i = 0; i < (UINT16)key_signature->public_key()->modulus().length(); i++) { + if (i % 32 == 0) bpInfo += UString("\n"); + bpInfo += usprintf("%02X", (UINT8)key_signature->public_key()->modulus().at(i)); + } + bpInfo += "\n"; + + // Calculate and add PubKey hashes + UINT8 hash[SHA384_HASH_SIZE]; + // SHA256 + sha256(key_signature->public_key()->modulus().data(), key_signature->public_key()->modulus().length() , hash); + bpInfo += UString("Boot Policy Public Key Hash (SHA256): "); + for (UINT8 i = 0; i < SHA256_HASH_SIZE; i++) { + bpInfo += usprintf("%02X", hash[i]); + } + bpInfo += "\n"; + bgBpHashSha256 = UByteArray((const char*)hash, SHA256_HASH_SIZE); + // SHA384 + sha384(key_signature->public_key()->modulus().data(), key_signature->public_key()->modulus().length() , hash); + bpInfo += UString("Boot Policy Public Key Hash (SHA384): "); + for (UINT8 i = 0; i < SHA384_HASH_SIZE; i++) { + bpInfo += usprintf("%02X", hash[i]); + } + bpInfo += "\n"; + bgBpHashSha384 = UByteArray((const char*)hash, SHA384_HASH_SIZE); + + // Add Signature + bpInfo += UString("Boot Policy Signature: "); + for (UINT16 i = 0; i < (UINT16)key_signature->signature()->signature().length(); i++) { + if (i % 32 == 0) bpInfo += UString("\n"); + bpInfo += usprintf("%02X", (UINT8)key_signature->signature()->signature().at(i)); + } + bpInfo += "\n"; + } + } + + securityInfo += bpInfo + "\n"; + bgBootPolicyFound = true; + return U_SUCCESS; + } + catch (...) { + // Do nothing here, will try parsing as v2 next + } + + // v2 + try { + is.seekg(localOffset, is.beg); + kaitai::kstream ks(&is); + intel_acbp_v2_t parsed(&ks); // This already verified the version to be >= 0x20 + // Valid BPM found + info = usprintf("LocalOffset: %08Xh, Version: %02Xh, BP SVN: %02Xh, ACM SVN: %02Xh", + localOffset, + parsed.version(), + parsed.bp_svn(), + parsed.acm_svn()); + + // Add BP header and body info + UString bpInfo + = usprintf("Intel BootGuard Boot Policy Manifest found at base %Xh\n" + "StructureId: '__ACBP__'\n" + "Version: %02Xh\n" + "HeaderSpecific: %02Xh\n" + "TotalSize: %04Xh\n" + "KeySignatureOffset: %04Xh\n" + "BPMRevision: %02Xh\n" + "BPSVN: %02Xh\n" + "ACMSVN: %02Xh\n" + "NEMDataSize: %04Xh\n", + model->base(parent) + localOffset, + parsed.version(), + parsed.header_specific(), + parsed.total_size(), + parsed.key_signature_offset(), + parsed.bpm_revision(), + parsed.bp_svn(), + parsed.acm_svn(), + parsed.nem_data_size()); + + bpInfo += UString("Boot Policy Elements:\n"); + const std::vector* elements = parsed.elements(); + for (intel_acbp_v2_t::acbp_element_t* element : *elements) { + const intel_acbp_v2_t::header_t* element_header = element->header(); + + UINT64 structure_id = element_header->structure_id(); + const char* structure_id_bytes = (const char*)&structure_id; + + bpInfo += usprintf("StructureId: '%c%c%c%c%c%c%c%c'\n" + "Version: %02Xh\n" + "HeaderSpecific: %02Xh\n" + "TotalSize: %04Xh\n", + structure_id_bytes[0], + structure_id_bytes[1], + structure_id_bytes[2], + structure_id_bytes[3], + structure_id_bytes[4], + structure_id_bytes[5], + structure_id_bytes[6], + structure_id_bytes[7], + element_header->version(), + element_header->header_specific(), + element_header->total_size()); + + // IBBS + if (element->_is_null_ibbs_body() == false) { + const intel_acbp_v2_t::ibbs_body_t* ibbs_body = element->ibbs_body(); + + // Valid IBBS element found + bpInfo += usprintf("SetNumber: %02Xh\n" + "PBETValue: %02Xh\n" + "Flags: %08Xh\n" + "MchBar: %016" PRIX64 "h\n" + "VtdBar: %016" PRIX64 "h\n" + "DmaProtectionBase0: %08Xh\n" + "DmaProtectionLimit0: %08Xh\n" + "DmaProtectionBase1: %016" PRIX64 "h\n" + "DmaProtectionLimit1: %016" PRIX64 "h\n" + "IbbEntryPoint: %08Xh\n" + "IbbDigestsSize: %02Xh\n" + "IbbDigestsCount: %02Xh\n" + "IbbSegmentsCount: %02Xh\n", + ibbs_body->set_number(), + ibbs_body->pbet_value(), + ibbs_body->flags(), + ibbs_body->mch_bar(), + ibbs_body->vtd_bar(), + ibbs_body->dma_protection_base0(), + ibbs_body->dma_protection_limit0(), + ibbs_body->dma_protection_base1(), + ibbs_body->dma_protection_limit1(), + ibbs_body->ibb_entry_point(), + ibbs_body->ibb_digests_size(), + ibbs_body->num_ibb_digests(), + ibbs_body->num_ibb_segments()); + + // Check for non-empty PostIbbHash + if (ibbs_body->post_ibb_digest()->len_hash() == 0) { + bpInfo += UString("PostIBB Hash: N/A\n"); + } + else { + // Add postIbbHash protected range + UByteArray postIbbHash(ibbs_body->post_ibb_digest()->hash().data(), ibbs_body->post_ibb_digest()->len_hash()); + if (postIbbHash.count('\x00') != postIbbHash.size() + && postIbbHash.count('\xFF') != postIbbHash.size()) { + PROTECTED_RANGE range = {}; + range.Type = PROTECTED_RANGE_INTEL_BOOT_GUARD_POST_IBB; + range.AlgorithmId = ibbs_body->post_ibb_digest()->hash_algorithm_id(); + range.Hash = postIbbHash; + ffsParser->protectedRanges.push_back(range); + } + + // Add PostIbbDigest + bpInfo += UString("PostIBB Hash (") + hashTypeToUString(ibbs_body->post_ibb_digest()->hash_algorithm_id()) + "): "; + for (UINT16 i = 0; i < ibbs_body->post_ibb_digest()->len_hash(); i++) { + bpInfo += usprintf("%02X", (UINT8)ibbs_body->post_ibb_digest()->hash().data()[i]); + } + bpInfo += "\n"; + } + + // Check for non-empty ObbHash + if (ibbs_body->obb_digest() == 0) { + bpInfo += UString("OBB Hash: N/A\n"); + } + else { + // Add ObbHash + bpInfo += UString("OBB Hash (") + hashTypeToUString(ibbs_body->obb_digest()->hash_algorithm_id()) + "): "; + for (UINT16 i = 0; i < ibbs_body->obb_digest()->len_hash(); i++) { + bpInfo += usprintf("%02X", (UINT8)ibbs_body->obb_digest()->hash().data()[i]); + } + bpInfo += "\n"; + + // Add ObbHash protected range + UByteArray obbHash(ibbs_body->obb_digest()->hash().data(), ibbs_body->obb_digest()->len_hash()); + if (obbHash.count('\x00') != obbHash.size() + && obbHash.count('\xFF') != obbHash.size()) { + PROTECTED_RANGE range = {}; + range.Type = PROTECTED_RANGE_INTEL_BOOT_GUARD_OBB; + range.AlgorithmId = ibbs_body->obb_digest()->hash_algorithm_id(); + range.Hash = obbHash; + ffsParser->protectedRanges.push_back(range); + } + } + + // Check for non-empty IbbDigests + if (ibbs_body->num_ibb_digests() == 0) { + bpInfo += UString("IBB Hashes: N/A\n"); + msg(usprintf("%s: Boot Policy without IBB digests", __FUNCTION__), parent); + } + else { + bpInfo += UString("IBB Hashes:\n"); + for (UINT16 i = 0; i < ibbs_body->num_ibb_digests(); i++) { + const intel_acbp_v2_t::hash_t* current_hash = ibbs_body->ibb_digests()->at(i); + bpInfo += hashTypeToUString(current_hash->hash_algorithm_id()) + ": "; + for (UINT16 j = 0; j < current_hash->len_hash(); j++) { + bpInfo += usprintf("%02X", (UINT8)current_hash->hash().data()[j]); + } + bpInfo += "\n"; + } + } + + // Check for non-empty IbbSegments + if (ibbs_body->num_ibb_segments() == 0) { + bpInfo += UString("IBB Segments: N/A\n"); + msg(usprintf("%s: Boot Policy without IBB segments", __FUNCTION__), parent); + } + else { + bpInfo += UString("IBB Segments:\n"); + for (UINT8 i = 0; i < ibbs_body->num_ibb_segments(); i++) { + const intel_acbp_v2_t::ibb_segment_t* current_segment = ibbs_body->ibb_segments()->at(i); + + bpInfo += usprintf("Flags: %04Xh, Address: %08Xh, Size: %08Xh\n", + current_segment->flags(), + current_segment->base(), + current_segment->size()); + + if (current_segment->flags() == intel_acbp_v2_t::IBB_SEGMENT_TYPE_IBB) { + PROTECTED_RANGE range = {}; + range.Offset = current_segment->base(); + range.Size =current_segment->size(); + range.Type = PROTECTED_RANGE_INTEL_BOOT_GUARD_IBB; + range.AlgorithmId = TCG_HASH_ALGORITHM_ID_SHA256; + ffsParser->protectedRanges.push_back(range); + } + } + } + } + // PMDA + else if (element->_is_null_pmda_body() == false) { + const intel_acbp_v2_t::pmda_body_t* pmda_body = element->pmda_body(); + + // Valid Microsoft PMDA element found + bpInfo += usprintf("TotalSize: %04Xh\n" + "Version: %08Xh\n" + "NumEntries: %08Xh\n", + pmda_body->total_size(), + pmda_body->version(), + pmda_body->num_entries()); + + if (pmda_body->num_entries() == 0) { + bpInfo += UString("PMDA Entries: N/A\n"); + } + else { + bpInfo += UString("PMDA Entries:\n"); + for (UINT32 i = 0; i < pmda_body->num_entries(); i++) { + const intel_acbp_v2_t::pmda_entry_v3_t* current_entry = pmda_body->entries()->at(i); + + UINT64 entry_id = current_entry->entry_id(); + const char* entry_id_bytes = (const char*)&entry_id; + + // Add element + bpInfo += usprintf("EntryId: '%c%c%c%c', Version: %04Xh, Address: %08Xh, Size: %08Xh\n", + entry_id_bytes[0], + entry_id_bytes[1], + entry_id_bytes[2], + entry_id_bytes[3], + current_entry->version(), + current_entry->base(), + current_entry->size()); + + // Add hash + bpInfo += hashTypeToUString(current_entry->hash()->hash_algorithm_id()) + ": "; + for (UINT16 j = 0; j < current_entry->hash()->len_hash(); j++) { + bpInfo += usprintf("%02X", (UINT8)current_entry->hash()->hash().data()[j]); + } + bpInfo += "\n"; + + // Add protected range + PROTECTED_RANGE range = {}; + range.Offset = current_entry->base(); + range.Size = current_entry->size(); + range.Type = PROTECTED_RANGE_VENDOR_HASH_MICROSOFT_PMDA; + range.AlgorithmId = current_entry->hash()->hash_algorithm_id(); + range.Hash = UByteArray(current_entry->hash()->hash().data(), current_entry->hash()->hash().size()); + ffsParser->protectedRanges.push_back(range); + } + } + } + bpInfo += "\n"; + } + + // Add Key Signature + const intel_acbp_v2_t::key_signature_t* key_signature = parsed.key_signature(); + bpInfo += usprintf("Boot Policy Key Signature:\n" + "Version: %02Xh\n" + "KeyId: %04Xh\n" + "SigScheme: %04Xh\n", + key_signature->version(), + key_signature->key_id(), + key_signature->sig_scheme()); + + // Add PubKey + bpInfo += usprintf("Boot Policy Public Key (Exponent: %Xh): ", key_signature->public_key()->exponent()); + for (UINT16 i = 0; i < (UINT16)key_signature->public_key()->modulus().length(); i++) { + if (i % 32 == 0) bpInfo += UString("\n"); + bpInfo += usprintf("%02X", (UINT8)key_signature->public_key()->modulus().at(i)); + } + bpInfo += "\n"; + + // Calculate and add PubKey hashes + UINT8 hash[SHA384_HASH_SIZE]; + // SHA256 + sha256(key_signature->public_key()->modulus().data(), key_signature->public_key()->modulus().length() , hash); + bpInfo += UString("Boot Policy Public Key Hash (SHA256): "); + for (UINT8 i = 0; i < SHA256_HASH_SIZE; i++) { + bpInfo += usprintf("%02X", hash[i]); + } + bpInfo += "\n"; + bgBpHashSha256 = UByteArray((const char*)hash, SHA256_HASH_SIZE); + // SHA384 + sha384(key_signature->public_key()->modulus().data(), key_signature->public_key()->modulus().length() , hash); + bpInfo += UString("Boot Policy Public Key Hash (SHA384): "); + for (UINT8 i = 0; i < SHA384_HASH_SIZE; i++) { + bpInfo += usprintf("%02X", hash[i]); + } + bpInfo += "\n"; + bgBpHashSha384 = UByteArray((const char*)hash, SHA384_HASH_SIZE); + + // Add Signature + bpInfo += UString("Boot Policy Signature: "); + for (UINT16 i = 0; i < (UINT16)key_signature->signature()->signature().length(); i++) { + if (i % 32 == 0) bpInfo += UString("\n"); + bpInfo += usprintf("%02X", (UINT8)key_signature->signature()->signature().at(i)); + } + bpInfo += "\n"; + + securityInfo += bpInfo + "\n"; + bgBootPolicyFound = true; + return U_SUCCESS; + } + catch (...) { + msg(usprintf("%s: unable to parse Boot Policy", __FUNCTION__), parent); + return U_INVALID_BOOT_GUARD_BOOT_POLICY; + } +} +#endif // U_ENABLE_ME_PARSING_SUPPORT diff --git a/common/fitparser.h b/common/fitparser.h new file mode 100644 index 0000000..d0d3553 --- /dev/null +++ b/common/fitparser.h @@ -0,0 +1,100 @@ +/* fitparser.h + +Copyright (c) 2022, Nikolaj Schlej. All rights reserved. + +This program and the accompanying materials +are licensed and made available under the terms and conditions of the BSD License +which accompanies this distribution. The full text of the license may be found at +http://opensource.org/licenses/bsd-license.php. + +THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, +WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. + +*/ + +#ifndef FITPARSER_H +#define FITPARSER_H + +#include + +#include "basetypes.h" +#include "ustring.h" +#include "ubytearray.h" +#include "treemodel.h" +#include "intel_fit.h" +#include "intel_microcode.h" +#include "ffsparser.h" + +class FfsParser; + +#ifdef U_ENABLE_FIT_PARSING_SUPPORT +class FitParser +{ +public: + // Default constructor and destructor + FitParser(TreeModel* treeModel, FfsParser* parser) : model(treeModel), ffsParser(parser), + bgAcmFound(false), bgKeyManifestFound(false), bgBootPolicyFound(false) {} + ~FitParser() {} + + // Returns messages + std::vector > getMessages() const { return messagesVector; } + // Clears messages + void clearMessages() { messagesVector.clear(); } + + // Obtain parsed FIT table + std::vector, UModelIndex> > getFitTable() const { return fitTable; } + + // Obtain security info + UString getSecurityInfo() const { return securityInfo; } + + // FIT parsing + USTATUS parseFit(const UModelIndex & index); + +private: + TreeModel *model; + FfsParser *ffsParser; + std::vector > messagesVector; + + std::vector, UModelIndex> > fitTable; + bool bgAcmFound; + bool bgKeyManifestFound; + bool bgBootPolicyFound; + UByteArray bgKmHash; + UByteArray bgBpHashSha256; + UByteArray bgBpHashSha384; + UString securityInfo; + + void msg(const UString message, const UModelIndex index = UModelIndex()) { + messagesVector.push_back(std::pair(message, index)); + } + + void findFitRecursive(const UModelIndex & index, UModelIndex & found, UINT32 & fitOffset); + USTATUS parseFitEntryMicrocode(const UByteArray & microcode, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize); + USTATUS parseFitEntryAcm(const UByteArray & acm, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize); + USTATUS parseFitEntryBootGuardKeyManifest(const UByteArray & keyManifest, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize); + USTATUS parseFitEntryBootGuardBootPolicy(const UByteArray & bootPolicy, const UINT32 localOffset, const UModelIndex & parent, UString & info, UINT32 &realSize); +}; +#else // U_ENABLE_FIT_PARSING_SUPPORT +class FitParser +{ +public: + // Default constructor and destructor + FitParser(TreeModel* treeModel, FfsParser* parser) { U_UNUSED_PARAMETER(treeModel); U_UNUSED_PARAMETER(parser); } + ~FitParser() {} + + // Returns messages + std::vector > getMessages() const { return std::vector >(); } + // Clears messages + void clearMessages() {} + + // Obtain parsed FIT table + std::vector, UModelIndex> > getFitTable() const { return std::vector, UModelIndex> >(); } + + // Obtain security info + UString getSecurityInfo() const { return UString(); } + + // FIT parsing + USTATUS parseFit(const UModelIndex & index) { U_UNUSED_PARAMETER(index); return U_SUCCESS; } +}; +#endif // U_ENABLE_FIT_PARSING_SUPPORT +#endif // FITPARSER_H diff --git a/common/generated/intel_acbp_v1.cpp b/common/generated/intel_acbp_v1.cpp new file mode 100644 index 0000000..4107fff --- /dev/null +++ b/common/generated/intel_acbp_v1.cpp @@ -0,0 +1,472 @@ +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "intel_acbp_v1.h" +#include "../kaitai/exceptions.h" + +intel_acbp_v1_t::intel_acbp_v1_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = this; (void)p__root; + m_elements = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::_read() { + m_structure_id = static_cast(m__io->read_u8le()); + if (!(structure_id() == intel_acbp_v1_t::STRUCTURE_IDS_ACBP)) { + throw kaitai::validation_not_equal_error(intel_acbp_v1_t::STRUCTURE_IDS_ACBP, structure_id(), _io(), std::string("/seq/0")); + } + m_version = m__io->read_u1(); + { + uint8_t _ = version(); + if (!(_ < 32)) { + throw kaitai::validation_expr_error(version(), _io(), std::string("/seq/1")); + } + } + m_reserved0 = m__io->read_u1(); + m_bpm_revision = m__io->read_u1(); + m_bp_svn = m__io->read_u1(); + m_acm_svn = m__io->read_u1(); + m_reserved1 = m__io->read_u1(); + m_nem_data_size = m__io->read_u2le(); + m_elements = new std::vector(); + { + int i = 0; + acbp_element_t* _; + do { + _ = new acbp_element_t(m__io, this, m__root); + m_elements->push_back(_); + i++; + } while (!( ((_->header()->structure_id() == intel_acbp_v1_t::STRUCTURE_IDS_PMSG) || (_io()->is_eof())) )); + } +} + +intel_acbp_v1_t::~intel_acbp_v1_t() { + _clean_up(); +} + +void intel_acbp_v1_t::_clean_up() { + if (m_elements) { + for (std::vector::iterator it = m_elements->begin(); it != m_elements->end(); ++it) { + delete *it; + } + delete m_elements; m_elements = 0; + } +} + +intel_acbp_v1_t::pmsg_body_t::pmsg_body_t(kaitai::kstream* p__io, intel_acbp_v1_t::acbp_element_t* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_public_key = 0; + m_signature = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::pmsg_body_t::_read() { + m_version = m__io->read_u1(); + m_key_id = m__io->read_u2le(); + m_public_key = new public_key_t(m__io, this, m__root); + m_sig_scheme = m__io->read_u2le(); + m_signature = new signature_t(m__io, this, m__root); +} + +intel_acbp_v1_t::pmsg_body_t::~pmsg_body_t() { + _clean_up(); +} + +void intel_acbp_v1_t::pmsg_body_t::_clean_up() { + if (m_public_key) { + delete m_public_key; m_public_key = 0; + } + if (m_signature) { + delete m_signature; m_signature = 0; + } +} + +intel_acbp_v1_t::acbp_element_t::acbp_element_t(kaitai::kstream* p__io, intel_acbp_v1_t* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_header = 0; + m_ibbs_body = 0; + m_pmda_body = 0; + m_pmsg_body = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::acbp_element_t::_read() { + m_header = new common_header_t(m__io, this, m__root); + n_ibbs_body = true; + if (header()->structure_id() == intel_acbp_v1_t::STRUCTURE_IDS_IBBS) { + n_ibbs_body = false; + m_ibbs_body = new ibbs_body_t(m__io, this, m__root); + } + n_pmda_body = true; + if (header()->structure_id() == intel_acbp_v1_t::STRUCTURE_IDS_PMDA) { + n_pmda_body = false; + m_pmda_body = new pmda_body_t(m__io, this, m__root); + } + n_pmsg_body = true; + if (header()->structure_id() == intel_acbp_v1_t::STRUCTURE_IDS_PMSG) { + n_pmsg_body = false; + m_pmsg_body = new pmsg_body_t(m__io, this, m__root); + } + n_invalid_body = true; + if ( ((header()->structure_id() != intel_acbp_v1_t::STRUCTURE_IDS_PMSG) && (header()->structure_id() != intel_acbp_v1_t::STRUCTURE_IDS_PMDA) && (header()->structure_id() != intel_acbp_v1_t::STRUCTURE_IDS_IBBS)) ) { + n_invalid_body = false; + m_invalid_body = m__io->read_bytes(0); + { + std::string _ = invalid_body(); + if (!(false)) { + throw kaitai::validation_expr_error(invalid_body(), _io(), std::string("/types/acbp_element/seq/4")); + } + } + } +} + +intel_acbp_v1_t::acbp_element_t::~acbp_element_t() { + _clean_up(); +} + +void intel_acbp_v1_t::acbp_element_t::_clean_up() { + if (m_header) { + delete m_header; m_header = 0; + } + if (!n_ibbs_body) { + if (m_ibbs_body) { + delete m_ibbs_body; m_ibbs_body = 0; + } + } + if (!n_pmda_body) { + if (m_pmda_body) { + delete m_pmda_body; m_pmda_body = 0; + } + } + if (!n_pmsg_body) { + if (m_pmsg_body) { + delete m_pmsg_body; m_pmsg_body = 0; + } + } + if (!n_invalid_body) { + } +} + +intel_acbp_v1_t::common_header_t::common_header_t(kaitai::kstream* p__io, intel_acbp_v1_t::acbp_element_t* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::common_header_t::_read() { + m_structure_id = static_cast(m__io->read_u8le()); + m_version = m__io->read_u1(); +} + +intel_acbp_v1_t::common_header_t::~common_header_t() { + _clean_up(); +} + +void intel_acbp_v1_t::common_header_t::_clean_up() { +} + +intel_acbp_v1_t::signature_t::signature_t(kaitai::kstream* p__io, intel_acbp_v1_t::pmsg_body_t* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::signature_t::_read() { + m_version = m__io->read_u1(); + m_size_bits = m__io->read_u2le(); + m_hash_algorithm_id = m__io->read_u2le(); + m_signature = m__io->read_bytes((size_bits() / 8)); +} + +intel_acbp_v1_t::signature_t::~signature_t() { + _clean_up(); +} + +void intel_acbp_v1_t::signature_t::_clean_up() { +} + +intel_acbp_v1_t::pmda_entry_v1_t::pmda_entry_v1_t(kaitai::kstream* p__io, intel_acbp_v1_t::pmda_body_t* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::pmda_entry_v1_t::_read() { + m_base = m__io->read_u4le(); + m_size = m__io->read_u4le(); + m_hash = m__io->read_bytes(32); +} + +intel_acbp_v1_t::pmda_entry_v1_t::~pmda_entry_v1_t() { + _clean_up(); +} + +void intel_acbp_v1_t::pmda_entry_v1_t::_clean_up() { +} + +intel_acbp_v1_t::ibb_segment_t::ibb_segment_t(kaitai::kstream* p__io, intel_acbp_v1_t::ibbs_body_t* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::ibb_segment_t::_read() { + m_reserved = m__io->read_u2le(); + m_flags = m__io->read_u2le(); + m_base = m__io->read_u4le(); + m_size = m__io->read_u4le(); +} + +intel_acbp_v1_t::ibb_segment_t::~ibb_segment_t() { + _clean_up(); +} + +void intel_acbp_v1_t::ibb_segment_t::_clean_up() { +} + +intel_acbp_v1_t::public_key_t::public_key_t(kaitai::kstream* p__io, intel_acbp_v1_t::pmsg_body_t* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::public_key_t::_read() { + m_version = m__io->read_u1(); + m_size_bits = m__io->read_u2le(); + m_exponent = m__io->read_u4le(); + m_modulus = m__io->read_bytes((size_bits() / 8)); +} + +intel_acbp_v1_t::public_key_t::~public_key_t() { + _clean_up(); +} + +void intel_acbp_v1_t::public_key_t::_clean_up() { +} + +intel_acbp_v1_t::hash_t::hash_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::hash_t::_read() { + m_hash_algorithm_id = m__io->read_u2le(); + m_len_hash = m__io->read_u2le(); + m_hash = m__io->read_bytes(32); +} + +intel_acbp_v1_t::hash_t::~hash_t() { + _clean_up(); +} + +void intel_acbp_v1_t::hash_t::_clean_up() { +} + +intel_acbp_v1_t::pmda_entry_v2_t::pmda_entry_v2_t(kaitai::kstream* p__io, intel_acbp_v1_t::pmda_body_t* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_hash = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::pmda_entry_v2_t::_read() { + m_base = m__io->read_u4le(); + m_size = m__io->read_u4le(); + m_hash = new hash_t(m__io, this, m__root); +} + +intel_acbp_v1_t::pmda_entry_v2_t::~pmda_entry_v2_t() { + _clean_up(); +} + +void intel_acbp_v1_t::pmda_entry_v2_t::_clean_up() { + if (m_hash) { + delete m_hash; m_hash = 0; + } +} + +intel_acbp_v1_t::ibbs_body_t::ibbs_body_t(kaitai::kstream* p__io, intel_acbp_v1_t::acbp_element_t* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_reserved = 0; + m_post_ibb_hash = 0; + m_ibb_hash = 0; + m_ibb_segments = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::ibbs_body_t::_read() { + m_reserved = new std::vector(); + const int l_reserved = 3; + for (int i = 0; i < l_reserved; i++) { + m_reserved->push_back(m__io->read_u1()); + } + m_flags = m__io->read_u4le(); + m_mch_bar = m__io->read_u8le(); + m_vtd_bar = m__io->read_u8le(); + m_dma_protection_base0 = m__io->read_u4le(); + m_dma_protection_limit0 = m__io->read_u4le(); + m_dma_protection_base1 = m__io->read_u8le(); + m_dma_protection_limit1 = m__io->read_u8le(); + m_post_ibb_hash = new hash_t(m__io, this, m__root); + m_ibb_entry_point = m__io->read_u4le(); + m_ibb_hash = new hash_t(m__io, this, m__root); + m_num_ibb_segments = m__io->read_u1(); + m_ibb_segments = new std::vector(); + const int l_ibb_segments = num_ibb_segments(); + for (int i = 0; i < l_ibb_segments; i++) { + m_ibb_segments->push_back(new ibb_segment_t(m__io, this, m__root)); + } +} + +intel_acbp_v1_t::ibbs_body_t::~ibbs_body_t() { + _clean_up(); +} + +void intel_acbp_v1_t::ibbs_body_t::_clean_up() { + if (m_reserved) { + delete m_reserved; m_reserved = 0; + } + if (m_post_ibb_hash) { + delete m_post_ibb_hash; m_post_ibb_hash = 0; + } + if (m_ibb_hash) { + delete m_ibb_hash; m_ibb_hash = 0; + } + if (m_ibb_segments) { + for (std::vector::iterator it = m_ibb_segments->begin(); it != m_ibb_segments->end(); ++it) { + delete *it; + } + delete m_ibb_segments; m_ibb_segments = 0; + } +} + +intel_acbp_v1_t::pmda_body_t::pmda_body_t(kaitai::kstream* p__io, intel_acbp_v1_t::acbp_element_t* p__parent, intel_acbp_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_entries_v1 = 0; + m_entries_v2 = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v1_t::pmda_body_t::_read() { + m_total_size = m__io->read_u2le(); + m_version = m__io->read_u4le(); + m_num_entries = m__io->read_u4le(); + n_entries_v1 = true; + if (version() == 1) { + n_entries_v1 = false; + m_entries_v1 = new std::vector(); + const int l_entries_v1 = num_entries(); + for (int i = 0; i < l_entries_v1; i++) { + m_entries_v1->push_back(new pmda_entry_v1_t(m__io, this, m__root)); + } + } + n_entries_v2 = true; + if (version() == 2) { + n_entries_v2 = false; + m_entries_v2 = new std::vector(); + const int l_entries_v2 = num_entries(); + for (int i = 0; i < l_entries_v2; i++) { + m_entries_v2->push_back(new pmda_entry_v2_t(m__io, this, m__root)); + } + } +} + +intel_acbp_v1_t::pmda_body_t::~pmda_body_t() { + _clean_up(); +} + +void intel_acbp_v1_t::pmda_body_t::_clean_up() { + if (!n_entries_v1) { + if (m_entries_v1) { + for (std::vector::iterator it = m_entries_v1->begin(); it != m_entries_v1->end(); ++it) { + delete *it; + } + delete m_entries_v1; m_entries_v1 = 0; + } + } + if (!n_entries_v2) { + if (m_entries_v2) { + for (std::vector::iterator it = m_entries_v2->begin(); it != m_entries_v2->end(); ++it) { + delete *it; + } + delete m_entries_v2; m_entries_v2 = 0; + } + } +} diff --git a/common/generated/intel_acbp_v1.h b/common/generated/intel_acbp_v1.h new file mode 100644 index 0000000..e142ef9 --- /dev/null +++ b/common/generated/intel_acbp_v1.h @@ -0,0 +1,457 @@ +#ifndef INTEL_ACBP_V1_H_ +#define INTEL_ACBP_V1_H_ + +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "../kaitai/kaitaistruct.h" +#include +#include + +#if KAITAI_STRUCT_VERSION < 9000L +#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required" +#endif + +class intel_acbp_v1_t : public kaitai::kstruct { + +public: + class pmsg_body_t; + class acbp_element_t; + class common_header_t; + class signature_t; + class pmda_entry_v1_t; + class ibb_segment_t; + class public_key_t; + class hash_t; + class pmda_entry_v2_t; + class ibbs_body_t; + class pmda_body_t; + + enum ibb_segment_type_t { + IBB_SEGMENT_TYPE_IBB = 0, + IBB_SEGMENT_TYPE_NON_IBB = 1 + }; + + enum structure_ids_t { + STRUCTURE_IDS_PMDA = 6872283318001360735LL, + STRUCTURE_IDS_PMSG = 6872289979495636831LL, + STRUCTURE_IDS_ACBP = 6872299801917087583LL, + STRUCTURE_IDS_IBBS = 6872303100435717983LL + }; + + intel_acbp_v1_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = 0, intel_acbp_v1_t* p__root = 0); + +private: + void _read(); + void _clean_up(); + +public: + ~intel_acbp_v1_t(); + + class pmsg_body_t : public kaitai::kstruct { + + public: + + pmsg_body_t(kaitai::kstream* p__io, intel_acbp_v1_t::acbp_element_t* p__parent = 0, intel_acbp_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~pmsg_body_t(); + + private: + uint8_t m_version; + uint16_t m_key_id; + public_key_t* m_public_key; + uint16_t m_sig_scheme; + signature_t* m_signature; + intel_acbp_v1_t* m__root; + intel_acbp_v1_t::acbp_element_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t key_id() const { return m_key_id; } + public_key_t* public_key() const { return m_public_key; } + uint16_t sig_scheme() const { return m_sig_scheme; } + signature_t* signature() const { return m_signature; } + intel_acbp_v1_t* _root() const { return m__root; } + intel_acbp_v1_t::acbp_element_t* _parent() const { return m__parent; } + }; + + class acbp_element_t : public kaitai::kstruct { + + public: + + acbp_element_t(kaitai::kstream* p__io, intel_acbp_v1_t* p__parent = 0, intel_acbp_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~acbp_element_t(); + + private: + common_header_t* m_header; + ibbs_body_t* m_ibbs_body; + bool n_ibbs_body; + + public: + bool _is_null_ibbs_body() { ibbs_body(); return n_ibbs_body; }; + + private: + pmda_body_t* m_pmda_body; + bool n_pmda_body; + + public: + bool _is_null_pmda_body() { pmda_body(); return n_pmda_body; }; + + private: + pmsg_body_t* m_pmsg_body; + bool n_pmsg_body; + + public: + bool _is_null_pmsg_body() { pmsg_body(); return n_pmsg_body; }; + + private: + std::string m_invalid_body; + bool n_invalid_body; + + public: + bool _is_null_invalid_body() { invalid_body(); return n_invalid_body; }; + + private: + intel_acbp_v1_t* m__root; + intel_acbp_v1_t* m__parent; + + public: + common_header_t* header() const { return m_header; } + ibbs_body_t* ibbs_body() const { return m_ibbs_body; } + pmda_body_t* pmda_body() const { return m_pmda_body; } + pmsg_body_t* pmsg_body() const { return m_pmsg_body; } + std::string invalid_body() const { return m_invalid_body; } + intel_acbp_v1_t* _root() const { return m__root; } + intel_acbp_v1_t* _parent() const { return m__parent; } + }; + + class common_header_t : public kaitai::kstruct { + + public: + + common_header_t(kaitai::kstream* p__io, intel_acbp_v1_t::acbp_element_t* p__parent = 0, intel_acbp_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~common_header_t(); + + private: + structure_ids_t m_structure_id; + uint8_t m_version; + intel_acbp_v1_t* m__root; + intel_acbp_v1_t::acbp_element_t* m__parent; + + public: + structure_ids_t structure_id() const { return m_structure_id; } + uint8_t version() const { return m_version; } + intel_acbp_v1_t* _root() const { return m__root; } + intel_acbp_v1_t::acbp_element_t* _parent() const { return m__parent; } + }; + + class signature_t : public kaitai::kstruct { + + public: + + signature_t(kaitai::kstream* p__io, intel_acbp_v1_t::pmsg_body_t* p__parent = 0, intel_acbp_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~signature_t(); + + private: + uint8_t m_version; + uint16_t m_size_bits; + uint16_t m_hash_algorithm_id; + std::string m_signature; + intel_acbp_v1_t* m__root; + intel_acbp_v1_t::pmsg_body_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t size_bits() const { return m_size_bits; } + uint16_t hash_algorithm_id() const { return m_hash_algorithm_id; } + std::string signature() const { return m_signature; } + intel_acbp_v1_t* _root() const { return m__root; } + intel_acbp_v1_t::pmsg_body_t* _parent() const { return m__parent; } + }; + + class pmda_entry_v1_t : public kaitai::kstruct { + + public: + + pmda_entry_v1_t(kaitai::kstream* p__io, intel_acbp_v1_t::pmda_body_t* p__parent = 0, intel_acbp_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~pmda_entry_v1_t(); + + private: + uint32_t m_base; + uint32_t m_size; + std::string m_hash; + intel_acbp_v1_t* m__root; + intel_acbp_v1_t::pmda_body_t* m__parent; + + public: + uint32_t base() const { return m_base; } + uint32_t size() const { return m_size; } + std::string hash() const { return m_hash; } + intel_acbp_v1_t* _root() const { return m__root; } + intel_acbp_v1_t::pmda_body_t* _parent() const { return m__parent; } + }; + + class ibb_segment_t : public kaitai::kstruct { + + public: + + ibb_segment_t(kaitai::kstream* p__io, intel_acbp_v1_t::ibbs_body_t* p__parent = 0, intel_acbp_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~ibb_segment_t(); + + private: + uint16_t m_reserved; + uint16_t m_flags; + uint32_t m_base; + uint32_t m_size; + intel_acbp_v1_t* m__root; + intel_acbp_v1_t::ibbs_body_t* m__parent; + + public: + uint16_t reserved() const { return m_reserved; } + uint16_t flags() const { return m_flags; } + uint32_t base() const { return m_base; } + uint32_t size() const { return m_size; } + intel_acbp_v1_t* _root() const { return m__root; } + intel_acbp_v1_t::ibbs_body_t* _parent() const { return m__parent; } + }; + + class public_key_t : public kaitai::kstruct { + + public: + + public_key_t(kaitai::kstream* p__io, intel_acbp_v1_t::pmsg_body_t* p__parent = 0, intel_acbp_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~public_key_t(); + + private: + uint8_t m_version; + uint16_t m_size_bits; + uint32_t m_exponent; + std::string m_modulus; + intel_acbp_v1_t* m__root; + intel_acbp_v1_t::pmsg_body_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t size_bits() const { return m_size_bits; } + uint32_t exponent() const { return m_exponent; } + std::string modulus() const { return m_modulus; } + intel_acbp_v1_t* _root() const { return m__root; } + intel_acbp_v1_t::pmsg_body_t* _parent() const { return m__parent; } + }; + + class hash_t : public kaitai::kstruct { + + public: + + hash_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = 0, intel_acbp_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~hash_t(); + + private: + uint16_t m_hash_algorithm_id; + uint16_t m_len_hash; + std::string m_hash; + intel_acbp_v1_t* m__root; + kaitai::kstruct* m__parent; + + public: + uint16_t hash_algorithm_id() const { return m_hash_algorithm_id; } + uint16_t len_hash() const { return m_len_hash; } + std::string hash() const { return m_hash; } + intel_acbp_v1_t* _root() const { return m__root; } + kaitai::kstruct* _parent() const { return m__parent; } + }; + + class pmda_entry_v2_t : public kaitai::kstruct { + + public: + + pmda_entry_v2_t(kaitai::kstream* p__io, intel_acbp_v1_t::pmda_body_t* p__parent = 0, intel_acbp_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~pmda_entry_v2_t(); + + private: + uint32_t m_base; + uint32_t m_size; + hash_t* m_hash; + intel_acbp_v1_t* m__root; + intel_acbp_v1_t::pmda_body_t* m__parent; + + public: + uint32_t base() const { return m_base; } + uint32_t size() const { return m_size; } + hash_t* hash() const { return m_hash; } + intel_acbp_v1_t* _root() const { return m__root; } + intel_acbp_v1_t::pmda_body_t* _parent() const { return m__parent; } + }; + + class ibbs_body_t : public kaitai::kstruct { + + public: + + ibbs_body_t(kaitai::kstream* p__io, intel_acbp_v1_t::acbp_element_t* p__parent = 0, intel_acbp_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~ibbs_body_t(); + + private: + std::vector* m_reserved; + uint32_t m_flags; + uint64_t m_mch_bar; + uint64_t m_vtd_bar; + uint32_t m_dma_protection_base0; + uint32_t m_dma_protection_limit0; + uint64_t m_dma_protection_base1; + uint64_t m_dma_protection_limit1; + hash_t* m_post_ibb_hash; + uint32_t m_ibb_entry_point; + hash_t* m_ibb_hash; + uint8_t m_num_ibb_segments; + std::vector* m_ibb_segments; + intel_acbp_v1_t* m__root; + intel_acbp_v1_t::acbp_element_t* m__parent; + + public: + std::vector* reserved() const { return m_reserved; } + uint32_t flags() const { return m_flags; } + uint64_t mch_bar() const { return m_mch_bar; } + uint64_t vtd_bar() const { return m_vtd_bar; } + uint32_t dma_protection_base0() const { return m_dma_protection_base0; } + uint32_t dma_protection_limit0() const { return m_dma_protection_limit0; } + uint64_t dma_protection_base1() const { return m_dma_protection_base1; } + uint64_t dma_protection_limit1() const { return m_dma_protection_limit1; } + hash_t* post_ibb_hash() const { return m_post_ibb_hash; } + uint32_t ibb_entry_point() const { return m_ibb_entry_point; } + hash_t* ibb_hash() const { return m_ibb_hash; } + uint8_t num_ibb_segments() const { return m_num_ibb_segments; } + std::vector* ibb_segments() const { return m_ibb_segments; } + intel_acbp_v1_t* _root() const { return m__root; } + intel_acbp_v1_t::acbp_element_t* _parent() const { return m__parent; } + }; + + class pmda_body_t : public kaitai::kstruct { + + public: + + pmda_body_t(kaitai::kstream* p__io, intel_acbp_v1_t::acbp_element_t* p__parent = 0, intel_acbp_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~pmda_body_t(); + + private: + uint16_t m_total_size; + uint32_t m_version; + uint32_t m_num_entries; + std::vector* m_entries_v1; + bool n_entries_v1; + + public: + bool _is_null_entries_v1() { entries_v1(); return n_entries_v1; }; + + private: + std::vector* m_entries_v2; + bool n_entries_v2; + + public: + bool _is_null_entries_v2() { entries_v2(); return n_entries_v2; }; + + private: + intel_acbp_v1_t* m__root; + intel_acbp_v1_t::acbp_element_t* m__parent; + + public: + uint16_t total_size() const { return m_total_size; } + uint32_t version() const { return m_version; } + uint32_t num_entries() const { return m_num_entries; } + std::vector* entries_v1() const { return m_entries_v1; } + std::vector* entries_v2() const { return m_entries_v2; } + intel_acbp_v1_t* _root() const { return m__root; } + intel_acbp_v1_t::acbp_element_t* _parent() const { return m__parent; } + }; + +private: + structure_ids_t m_structure_id; + uint8_t m_version; + uint8_t m_reserved0; + uint8_t m_bpm_revision; + uint8_t m_bp_svn; + uint8_t m_acm_svn; + uint8_t m_reserved1; + uint16_t m_nem_data_size; + std::vector* m_elements; + intel_acbp_v1_t* m__root; + kaitai::kstruct* m__parent; + +public: + structure_ids_t structure_id() const { return m_structure_id; } + uint8_t version() const { return m_version; } + uint8_t reserved0() const { return m_reserved0; } + uint8_t bpm_revision() const { return m_bpm_revision; } + uint8_t bp_svn() const { return m_bp_svn; } + uint8_t acm_svn() const { return m_acm_svn; } + uint8_t reserved1() const { return m_reserved1; } + uint16_t nem_data_size() const { return m_nem_data_size; } + std::vector* elements() const { return m_elements; } + intel_acbp_v1_t* _root() const { return m__root; } + kaitai::kstruct* _parent() const { return m__parent; } +}; + +#endif // INTEL_ACBP_V1_H_ diff --git a/common/generated/intel_acbp_v2.cpp b/common/generated/intel_acbp_v2.cpp new file mode 100644 index 0000000..2684f9d --- /dev/null +++ b/common/generated/intel_acbp_v2.cpp @@ -0,0 +1,443 @@ +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "intel_acbp_v2.h" +#include "../kaitai/exceptions.h" + +intel_acbp_v2_t::intel_acbp_v2_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, intel_acbp_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = this; (void)p__root; + m_elements = 0; + m_key_signature = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v2_t::_read() { + m_structure_id = static_cast(m__io->read_u8le()); + if (!(structure_id() == intel_acbp_v2_t::STRUCTURE_IDS_ACBP)) { + throw kaitai::validation_not_equal_error(intel_acbp_v2_t::STRUCTURE_IDS_ACBP, structure_id(), _io(), std::string("/seq/0")); + } + m_version = m__io->read_u1(); + { + uint8_t _ = version(); + if (!(_ >= 32)) { + throw kaitai::validation_expr_error(version(), _io(), std::string("/seq/1")); + } + } + m_header_specific = m__io->read_u1(); + m_total_size = m__io->read_u2le(); + if (!(total_size() == 20)) { + throw kaitai::validation_not_equal_error(20, total_size(), _io(), std::string("/seq/3")); + } + m_key_signature_offset = m__io->read_u2le(); + m_bpm_revision = m__io->read_u1(); + m_bp_svn = m__io->read_u1(); + m_acm_svn = m__io->read_u1(); + m_reserved = m__io->read_u1(); + m_nem_data_size = m__io->read_u2le(); + m_elements = new std::vector(); + { + int i = 0; + acbp_element_t* _; + do { + _ = new acbp_element_t(m__io, this, m__root); + m_elements->push_back(_); + i++; + } while (!( ((_->header()->total_size() == 0) || (_->header()->structure_id() == intel_acbp_v2_t::STRUCTURE_IDS_PMSG)) )); + } + m_key_signature = new key_signature_t(m__io, this, m__root); +} + +intel_acbp_v2_t::~intel_acbp_v2_t() { + _clean_up(); +} + +void intel_acbp_v2_t::_clean_up() { + if (m_elements) { + for (std::vector::iterator it = m_elements->begin(); it != m_elements->end(); ++it) { + delete *it; + } + delete m_elements; m_elements = 0; + } + if (m_key_signature) { + delete m_key_signature; m_key_signature = 0; + } +} + +intel_acbp_v2_t::acbp_element_t::acbp_element_t(kaitai::kstream* p__io, intel_acbp_v2_t* p__parent, intel_acbp_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_header = 0; + m_ibbs_body = 0; + m_pmda_body = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v2_t::acbp_element_t::_read() { + m_header = new header_t(m__io, this, m__root); + n_ibbs_body = true; + if ( ((header()->structure_id() == intel_acbp_v2_t::STRUCTURE_IDS_IBBS) && (header()->total_size() >= 12)) ) { + n_ibbs_body = false; + m_ibbs_body = new ibbs_body_t(m__io, this, m__root); + } + n_pmda_body = true; + if ( ((header()->structure_id() == intel_acbp_v2_t::STRUCTURE_IDS_PMDA) && (header()->total_size() >= 12)) ) { + n_pmda_body = false; + m_pmda_body = new pmda_body_t(m__io, this, m__root); + } + n_generic_body = true; + if ( ((header()->structure_id() != intel_acbp_v2_t::STRUCTURE_IDS_IBBS) && (header()->structure_id() != intel_acbp_v2_t::STRUCTURE_IDS_PMDA) && (header()->total_size() >= 12)) ) { + n_generic_body = false; + m_generic_body = m__io->read_bytes((header()->total_size() - 12)); + } +} + +intel_acbp_v2_t::acbp_element_t::~acbp_element_t() { + _clean_up(); +} + +void intel_acbp_v2_t::acbp_element_t::_clean_up() { + if (m_header) { + delete m_header; m_header = 0; + } + if (!n_ibbs_body) { + if (m_ibbs_body) { + delete m_ibbs_body; m_ibbs_body = 0; + } + } + if (!n_pmda_body) { + if (m_pmda_body) { + delete m_pmda_body; m_pmda_body = 0; + } + } + if (!n_generic_body) { + } +} + +intel_acbp_v2_t::key_signature_t::key_signature_t(kaitai::kstream* p__io, intel_acbp_v2_t* p__parent, intel_acbp_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_public_key = 0; + m_signature = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v2_t::key_signature_t::_read() { + m_version = m__io->read_u1(); + m_key_id = m__io->read_u2le(); + m_public_key = new public_key_t(m__io, this, m__root); + m_sig_scheme = m__io->read_u2le(); + m_signature = new signature_t(m__io, this, m__root); +} + +intel_acbp_v2_t::key_signature_t::~key_signature_t() { + _clean_up(); +} + +void intel_acbp_v2_t::key_signature_t::_clean_up() { + if (m_public_key) { + delete m_public_key; m_public_key = 0; + } + if (m_signature) { + delete m_signature; m_signature = 0; + } +} + +intel_acbp_v2_t::signature_t::signature_t(kaitai::kstream* p__io, intel_acbp_v2_t::key_signature_t* p__parent, intel_acbp_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v2_t::signature_t::_read() { + m_version = m__io->read_u1(); + m_size_bits = m__io->read_u2le(); + m_hash_algorithm_id = m__io->read_u2le(); + m_signature = m__io->read_bytes((size_bits() / 8)); +} + +intel_acbp_v2_t::signature_t::~signature_t() { + _clean_up(); +} + +void intel_acbp_v2_t::signature_t::_clean_up() { +} + +intel_acbp_v2_t::ibb_segment_t::ibb_segment_t(kaitai::kstream* p__io, intel_acbp_v2_t::ibbs_body_t* p__parent, intel_acbp_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v2_t::ibb_segment_t::_read() { + m_reserved = m__io->read_u2le(); + m_flags = m__io->read_u2le(); + m_base = m__io->read_u4le(); + m_size = m__io->read_u4le(); +} + +intel_acbp_v2_t::ibb_segment_t::~ibb_segment_t() { + _clean_up(); +} + +void intel_acbp_v2_t::ibb_segment_t::_clean_up() { +} + +intel_acbp_v2_t::public_key_t::public_key_t(kaitai::kstream* p__io, intel_acbp_v2_t::key_signature_t* p__parent, intel_acbp_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v2_t::public_key_t::_read() { + m_version = m__io->read_u1(); + m_size_bits = m__io->read_u2le(); + m_exponent = m__io->read_u4le(); + m_modulus = m__io->read_bytes((size_bits() / 8)); +} + +intel_acbp_v2_t::public_key_t::~public_key_t() { + _clean_up(); +} + +void intel_acbp_v2_t::public_key_t::_clean_up() { +} + +intel_acbp_v2_t::hash_t::hash_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, intel_acbp_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v2_t::hash_t::_read() { + m_hash_algorithm_id = m__io->read_u2le(); + m_len_hash = m__io->read_u2le(); + m_hash = m__io->read_bytes(len_hash()); +} + +intel_acbp_v2_t::hash_t::~hash_t() { + _clean_up(); +} + +void intel_acbp_v2_t::hash_t::_clean_up() { +} + +intel_acbp_v2_t::header_t::header_t(kaitai::kstream* p__io, intel_acbp_v2_t::acbp_element_t* p__parent, intel_acbp_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v2_t::header_t::_read() { + m_structure_id = static_cast(m__io->read_u8le()); + m_version = m__io->read_u1(); + m_header_specific = m__io->read_u1(); + m_total_size = m__io->read_u2le(); +} + +intel_acbp_v2_t::header_t::~header_t() { + _clean_up(); +} + +void intel_acbp_v2_t::header_t::_clean_up() { +} + +intel_acbp_v2_t::pmda_entry_v3_t::pmda_entry_v3_t(kaitai::kstream* p__io, intel_acbp_v2_t::pmda_body_t* p__parent, intel_acbp_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_hash = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v2_t::pmda_entry_v3_t::_read() { + m_entry_id = m__io->read_u4le(); + m_base = m__io->read_u4le(); + m_size = m__io->read_u4le(); + m_total_entry_size = m__io->read_u2le(); + m_version = m__io->read_u2le(); + if (!(version() == 3)) { + throw kaitai::validation_not_equal_error(3, version(), _io(), std::string("/types/pmda_entry_v3/seq/4")); + } + m_hash = new hash_t(m__io, this, m__root); +} + +intel_acbp_v2_t::pmda_entry_v3_t::~pmda_entry_v3_t() { + _clean_up(); +} + +void intel_acbp_v2_t::pmda_entry_v3_t::_clean_up() { + if (m_hash) { + delete m_hash; m_hash = 0; + } +} + +intel_acbp_v2_t::ibbs_body_t::ibbs_body_t(kaitai::kstream* p__io, intel_acbp_v2_t::acbp_element_t* p__parent, intel_acbp_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_post_ibb_digest = 0; + m_ibb_digests = 0; + m_obb_digest = 0; + m_reserved2 = 0; + m_ibb_segments = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v2_t::ibbs_body_t::_read() { + m_reserved0 = m__io->read_u1(); + m_set_number = m__io->read_u1(); + m_reserved1 = m__io->read_u1(); + m_pbet_value = m__io->read_u1(); + m_flags = m__io->read_u4le(); + m_mch_bar = m__io->read_u8le(); + m_vtd_bar = m__io->read_u8le(); + m_dma_protection_base0 = m__io->read_u4le(); + m_dma_protection_limit0 = m__io->read_u4le(); + m_dma_protection_base1 = m__io->read_u8le(); + m_dma_protection_limit1 = m__io->read_u8le(); + m_post_ibb_digest = new hash_t(m__io, this, m__root); + m_ibb_entry_point = m__io->read_u4le(); + m_ibb_digests_size = m__io->read_u2le(); + m_num_ibb_digests = m__io->read_u2le(); + m_ibb_digests = new std::vector(); + const int l_ibb_digests = num_ibb_digests(); + for (int i = 0; i < l_ibb_digests; i++) { + m_ibb_digests->push_back(new hash_t(m__io, this, m__root)); + } + m_obb_digest = new hash_t(m__io, this, m__root); + m_reserved2 = new std::vector(); + const int l_reserved2 = 3; + for (int i = 0; i < l_reserved2; i++) { + m_reserved2->push_back(m__io->read_u1()); + } + m_num_ibb_segments = m__io->read_u1(); + m_ibb_segments = new std::vector(); + const int l_ibb_segments = num_ibb_segments(); + for (int i = 0; i < l_ibb_segments; i++) { + m_ibb_segments->push_back(new ibb_segment_t(m__io, this, m__root)); + } +} + +intel_acbp_v2_t::ibbs_body_t::~ibbs_body_t() { + _clean_up(); +} + +void intel_acbp_v2_t::ibbs_body_t::_clean_up() { + if (m_post_ibb_digest) { + delete m_post_ibb_digest; m_post_ibb_digest = 0; + } + if (m_ibb_digests) { + for (std::vector::iterator it = m_ibb_digests->begin(); it != m_ibb_digests->end(); ++it) { + delete *it; + } + delete m_ibb_digests; m_ibb_digests = 0; + } + if (m_obb_digest) { + delete m_obb_digest; m_obb_digest = 0; + } + if (m_reserved2) { + delete m_reserved2; m_reserved2 = 0; + } + if (m_ibb_segments) { + for (std::vector::iterator it = m_ibb_segments->begin(); it != m_ibb_segments->end(); ++it) { + delete *it; + } + delete m_ibb_segments; m_ibb_segments = 0; + } +} + +intel_acbp_v2_t::pmda_body_t::pmda_body_t(kaitai::kstream* p__io, intel_acbp_v2_t::acbp_element_t* p__parent, intel_acbp_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_entries = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acbp_v2_t::pmda_body_t::_read() { + m_reserved = m__io->read_u2le(); + m_total_size = m__io->read_u2le(); + m_version = m__io->read_u4le(); + m_num_entries = m__io->read_u4le(); + m_entries = new std::vector(); + const int l_entries = num_entries(); + for (int i = 0; i < l_entries; i++) { + m_entries->push_back(new pmda_entry_v3_t(m__io, this, m__root)); + } +} + +intel_acbp_v2_t::pmda_body_t::~pmda_body_t() { + _clean_up(); +} + +void intel_acbp_v2_t::pmda_body_t::_clean_up() { + if (m_entries) { + for (std::vector::iterator it = m_entries->begin(); it != m_entries->end(); ++it) { + delete *it; + } + delete m_entries; m_entries = 0; + } +} diff --git a/common/generated/intel_acbp_v2.h b/common/generated/intel_acbp_v2.h new file mode 100644 index 0000000..59ce1c4 --- /dev/null +++ b/common/generated/intel_acbp_v2.h @@ -0,0 +1,441 @@ +#ifndef INTEL_ACBP_V2_H_ +#define INTEL_ACBP_V2_H_ + +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "../kaitai/kaitaistruct.h" +#include +#include + +#if KAITAI_STRUCT_VERSION < 9000L +#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required" +#endif + +class intel_acbp_v2_t : public kaitai::kstruct { + +public: + class acbp_element_t; + class key_signature_t; + class signature_t; + class ibb_segment_t; + class public_key_t; + class hash_t; + class header_t; + class pmda_entry_v3_t; + class ibbs_body_t; + class pmda_body_t; + + enum ibb_segment_type_t { + IBB_SEGMENT_TYPE_IBB = 0, + IBB_SEGMENT_TYPE_NON_IBB = 1 + }; + + enum structure_ids_t { + STRUCTURE_IDS_PMDA = 6872283318001360735LL, + STRUCTURE_IDS_PMSG = 6872289979495636831LL, + STRUCTURE_IDS_ACBP = 6872299801917087583LL, + STRUCTURE_IDS_IBBS = 6872303100435717983LL, + STRUCTURE_IDS_PCDS = 6872303109042888543LL, + STRUCTURE_IDS_PFRS = 6872303169222762335LL, + STRUCTURE_IDS_TXTS = 6872303178114948959LL + }; + + intel_acbp_v2_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = 0, intel_acbp_v2_t* p__root = 0); + +private: + void _read(); + void _clean_up(); + +public: + ~intel_acbp_v2_t(); + + class acbp_element_t : public kaitai::kstruct { + + public: + + acbp_element_t(kaitai::kstream* p__io, intel_acbp_v2_t* p__parent = 0, intel_acbp_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~acbp_element_t(); + + private: + header_t* m_header; + ibbs_body_t* m_ibbs_body; + bool n_ibbs_body; + + public: + bool _is_null_ibbs_body() { ibbs_body(); return n_ibbs_body; }; + + private: + pmda_body_t* m_pmda_body; + bool n_pmda_body; + + public: + bool _is_null_pmda_body() { pmda_body(); return n_pmda_body; }; + + private: + std::string m_generic_body; + bool n_generic_body; + + public: + bool _is_null_generic_body() { generic_body(); return n_generic_body; }; + + private: + intel_acbp_v2_t* m__root; + intel_acbp_v2_t* m__parent; + + public: + header_t* header() const { return m_header; } + ibbs_body_t* ibbs_body() const { return m_ibbs_body; } + pmda_body_t* pmda_body() const { return m_pmda_body; } + std::string generic_body() const { return m_generic_body; } + intel_acbp_v2_t* _root() const { return m__root; } + intel_acbp_v2_t* _parent() const { return m__parent; } + }; + + class key_signature_t : public kaitai::kstruct { + + public: + + key_signature_t(kaitai::kstream* p__io, intel_acbp_v2_t* p__parent = 0, intel_acbp_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~key_signature_t(); + + private: + uint8_t m_version; + uint16_t m_key_id; + public_key_t* m_public_key; + uint16_t m_sig_scheme; + signature_t* m_signature; + intel_acbp_v2_t* m__root; + intel_acbp_v2_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t key_id() const { return m_key_id; } + public_key_t* public_key() const { return m_public_key; } + uint16_t sig_scheme() const { return m_sig_scheme; } + signature_t* signature() const { return m_signature; } + intel_acbp_v2_t* _root() const { return m__root; } + intel_acbp_v2_t* _parent() const { return m__parent; } + }; + + class signature_t : public kaitai::kstruct { + + public: + + signature_t(kaitai::kstream* p__io, intel_acbp_v2_t::key_signature_t* p__parent = 0, intel_acbp_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~signature_t(); + + private: + uint8_t m_version; + uint16_t m_size_bits; + uint16_t m_hash_algorithm_id; + std::string m_signature; + intel_acbp_v2_t* m__root; + intel_acbp_v2_t::key_signature_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t size_bits() const { return m_size_bits; } + uint16_t hash_algorithm_id() const { return m_hash_algorithm_id; } + std::string signature() const { return m_signature; } + intel_acbp_v2_t* _root() const { return m__root; } + intel_acbp_v2_t::key_signature_t* _parent() const { return m__parent; } + }; + + class ibb_segment_t : public kaitai::kstruct { + + public: + + ibb_segment_t(kaitai::kstream* p__io, intel_acbp_v2_t::ibbs_body_t* p__parent = 0, intel_acbp_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~ibb_segment_t(); + + private: + uint16_t m_reserved; + uint16_t m_flags; + uint32_t m_base; + uint32_t m_size; + intel_acbp_v2_t* m__root; + intel_acbp_v2_t::ibbs_body_t* m__parent; + + public: + uint16_t reserved() const { return m_reserved; } + uint16_t flags() const { return m_flags; } + uint32_t base() const { return m_base; } + uint32_t size() const { return m_size; } + intel_acbp_v2_t* _root() const { return m__root; } + intel_acbp_v2_t::ibbs_body_t* _parent() const { return m__parent; } + }; + + class public_key_t : public kaitai::kstruct { + + public: + + public_key_t(kaitai::kstream* p__io, intel_acbp_v2_t::key_signature_t* p__parent = 0, intel_acbp_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~public_key_t(); + + private: + uint8_t m_version; + uint16_t m_size_bits; + uint32_t m_exponent; + std::string m_modulus; + intel_acbp_v2_t* m__root; + intel_acbp_v2_t::key_signature_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t size_bits() const { return m_size_bits; } + uint32_t exponent() const { return m_exponent; } + std::string modulus() const { return m_modulus; } + intel_acbp_v2_t* _root() const { return m__root; } + intel_acbp_v2_t::key_signature_t* _parent() const { return m__parent; } + }; + + class hash_t : public kaitai::kstruct { + + public: + + hash_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = 0, intel_acbp_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~hash_t(); + + private: + uint16_t m_hash_algorithm_id; + uint16_t m_len_hash; + std::string m_hash; + intel_acbp_v2_t* m__root; + kaitai::kstruct* m__parent; + + public: + uint16_t hash_algorithm_id() const { return m_hash_algorithm_id; } + uint16_t len_hash() const { return m_len_hash; } + std::string hash() const { return m_hash; } + intel_acbp_v2_t* _root() const { return m__root; } + kaitai::kstruct* _parent() const { return m__parent; } + }; + + class header_t : public kaitai::kstruct { + + public: + + header_t(kaitai::kstream* p__io, intel_acbp_v2_t::acbp_element_t* p__parent = 0, intel_acbp_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~header_t(); + + private: + structure_ids_t m_structure_id; + uint8_t m_version; + uint8_t m_header_specific; + uint16_t m_total_size; + intel_acbp_v2_t* m__root; + intel_acbp_v2_t::acbp_element_t* m__parent; + + public: + structure_ids_t structure_id() const { return m_structure_id; } + uint8_t version() const { return m_version; } + uint8_t header_specific() const { return m_header_specific; } + uint16_t total_size() const { return m_total_size; } + intel_acbp_v2_t* _root() const { return m__root; } + intel_acbp_v2_t::acbp_element_t* _parent() const { return m__parent; } + }; + + class pmda_entry_v3_t : public kaitai::kstruct { + + public: + + pmda_entry_v3_t(kaitai::kstream* p__io, intel_acbp_v2_t::pmda_body_t* p__parent = 0, intel_acbp_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~pmda_entry_v3_t(); + + private: + uint32_t m_entry_id; + uint32_t m_base; + uint32_t m_size; + uint16_t m_total_entry_size; + uint16_t m_version; + hash_t* m_hash; + intel_acbp_v2_t* m__root; + intel_acbp_v2_t::pmda_body_t* m__parent; + + public: + uint32_t entry_id() const { return m_entry_id; } + uint32_t base() const { return m_base; } + uint32_t size() const { return m_size; } + uint16_t total_entry_size() const { return m_total_entry_size; } + uint16_t version() const { return m_version; } + hash_t* hash() const { return m_hash; } + intel_acbp_v2_t* _root() const { return m__root; } + intel_acbp_v2_t::pmda_body_t* _parent() const { return m__parent; } + }; + + class ibbs_body_t : public kaitai::kstruct { + + public: + + ibbs_body_t(kaitai::kstream* p__io, intel_acbp_v2_t::acbp_element_t* p__parent = 0, intel_acbp_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~ibbs_body_t(); + + private: + uint8_t m_reserved0; + uint8_t m_set_number; + uint8_t m_reserved1; + uint8_t m_pbet_value; + uint32_t m_flags; + uint64_t m_mch_bar; + uint64_t m_vtd_bar; + uint32_t m_dma_protection_base0; + uint32_t m_dma_protection_limit0; + uint64_t m_dma_protection_base1; + uint64_t m_dma_protection_limit1; + hash_t* m_post_ibb_digest; + uint32_t m_ibb_entry_point; + uint16_t m_ibb_digests_size; + uint16_t m_num_ibb_digests; + std::vector* m_ibb_digests; + hash_t* m_obb_digest; + std::vector* m_reserved2; + uint8_t m_num_ibb_segments; + std::vector* m_ibb_segments; + intel_acbp_v2_t* m__root; + intel_acbp_v2_t::acbp_element_t* m__parent; + + public: + uint8_t reserved0() const { return m_reserved0; } + uint8_t set_number() const { return m_set_number; } + uint8_t reserved1() const { return m_reserved1; } + uint8_t pbet_value() const { return m_pbet_value; } + uint32_t flags() const { return m_flags; } + uint64_t mch_bar() const { return m_mch_bar; } + uint64_t vtd_bar() const { return m_vtd_bar; } + uint32_t dma_protection_base0() const { return m_dma_protection_base0; } + uint32_t dma_protection_limit0() const { return m_dma_protection_limit0; } + uint64_t dma_protection_base1() const { return m_dma_protection_base1; } + uint64_t dma_protection_limit1() const { return m_dma_protection_limit1; } + hash_t* post_ibb_digest() const { return m_post_ibb_digest; } + uint32_t ibb_entry_point() const { return m_ibb_entry_point; } + uint16_t ibb_digests_size() const { return m_ibb_digests_size; } + uint16_t num_ibb_digests() const { return m_num_ibb_digests; } + std::vector* ibb_digests() const { return m_ibb_digests; } + hash_t* obb_digest() const { return m_obb_digest; } + std::vector* reserved2() const { return m_reserved2; } + uint8_t num_ibb_segments() const { return m_num_ibb_segments; } + std::vector* ibb_segments() const { return m_ibb_segments; } + intel_acbp_v2_t* _root() const { return m__root; } + intel_acbp_v2_t::acbp_element_t* _parent() const { return m__parent; } + }; + + class pmda_body_t : public kaitai::kstruct { + + public: + + pmda_body_t(kaitai::kstream* p__io, intel_acbp_v2_t::acbp_element_t* p__parent = 0, intel_acbp_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~pmda_body_t(); + + private: + uint16_t m_reserved; + uint16_t m_total_size; + uint32_t m_version; + uint32_t m_num_entries; + std::vector* m_entries; + intel_acbp_v2_t* m__root; + intel_acbp_v2_t::acbp_element_t* m__parent; + + public: + uint16_t reserved() const { return m_reserved; } + uint16_t total_size() const { return m_total_size; } + uint32_t version() const { return m_version; } + uint32_t num_entries() const { return m_num_entries; } + std::vector* entries() const { return m_entries; } + intel_acbp_v2_t* _root() const { return m__root; } + intel_acbp_v2_t::acbp_element_t* _parent() const { return m__parent; } + }; + +private: + structure_ids_t m_structure_id; + uint8_t m_version; + uint8_t m_header_specific; + uint16_t m_total_size; + uint16_t m_key_signature_offset; + uint8_t m_bpm_revision; + uint8_t m_bp_svn; + uint8_t m_acm_svn; + uint8_t m_reserved; + uint16_t m_nem_data_size; + std::vector* m_elements; + key_signature_t* m_key_signature; + intel_acbp_v2_t* m__root; + kaitai::kstruct* m__parent; + +public: + structure_ids_t structure_id() const { return m_structure_id; } + uint8_t version() const { return m_version; } + uint8_t header_specific() const { return m_header_specific; } + uint16_t total_size() const { return m_total_size; } + uint16_t key_signature_offset() const { return m_key_signature_offset; } + uint8_t bpm_revision() const { return m_bpm_revision; } + uint8_t bp_svn() const { return m_bp_svn; } + uint8_t acm_svn() const { return m_acm_svn; } + uint8_t reserved() const { return m_reserved; } + uint16_t nem_data_size() const { return m_nem_data_size; } + std::vector* elements() const { return m_elements; } + key_signature_t* key_signature() const { return m_key_signature; } + intel_acbp_v2_t* _root() const { return m__root; } + kaitai::kstruct* _parent() const { return m__parent; } +}; + +#endif // INTEL_ACBP_V2_H_ diff --git a/common/generated/intel_acm.cpp b/common/generated/intel_acm.cpp new file mode 100644 index 0000000..78fd1eb --- /dev/null +++ b/common/generated/intel_acm.cpp @@ -0,0 +1,92 @@ +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "intel_acm.h" +#include "../kaitai/exceptions.h" + +intel_acm_t::intel_acm_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, intel_acm_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = this; (void)p__root; + m_header = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acm_t::_read() { + m_header = new header_t(m__io, this, m__root); + m_body = m__io->read_bytes((4 * ((header()->module_size() - header()->header_size()) - header()->scratch_space_size()))); +} + +intel_acm_t::~intel_acm_t() { + _clean_up(); +} + +void intel_acm_t::_clean_up() { + if (m_header) { + delete m_header; m_header = 0; + } +} + +intel_acm_t::header_t::header_t(kaitai::kstream* p__io, intel_acm_t* p__parent, intel_acm_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_acm_t::header_t::_read() { + m_module_type = m__io->read_u2le(); + if (!(module_type() == 2)) { + throw kaitai::validation_not_equal_error(2, module_type(), _io(), std::string("/types/header/seq/0")); + } + m_module_subtype = static_cast(m__io->read_u2le()); + m_header_size = m__io->read_u4le(); + m_header_version = m__io->read_u4le(); + m_chipset_id = m__io->read_u2le(); + m_flags = m__io->read_u2le(); + m_module_vendor = m__io->read_u4le(); + if (!(module_vendor() == 32902)) { + throw kaitai::validation_not_equal_error(32902, module_vendor(), _io(), std::string("/types/header/seq/6")); + } + m_date_day = m__io->read_u1(); + m_date_month = m__io->read_u1(); + m_date_year = m__io->read_u2le(); + m_module_size = m__io->read_u4le(); + m_acm_svn = m__io->read_u2le(); + m_se_svn = m__io->read_u2le(); + m_code_control_flags = m__io->read_u4le(); + m_error_entry_point = m__io->read_u4le(); + m_gdt_max = m__io->read_u4le(); + m_gdt_base = m__io->read_u4le(); + m_segment_sel = m__io->read_u4le(); + m_entry_point = m__io->read_u4le(); + m_reserved = m__io->read_bytes(64); + m_key_size = m__io->read_u4le(); + m_scratch_space_size = m__io->read_u4le(); + m_rsa_public_key = m__io->read_bytes((4 * key_size())); + n_rsa_exponent = true; + if (header_version() == 0) { + n_rsa_exponent = false; + m_rsa_exponent = m__io->read_u4le(); + } + m_rsa_signature = m__io->read_bytes((4 * key_size())); + m_scratch_space = m__io->read_bytes((4 * scratch_space_size())); +} + +intel_acm_t::header_t::~header_t() { + _clean_up(); +} + +void intel_acm_t::header_t::_clean_up() { + if (!n_rsa_exponent) { + } +} diff --git a/common/generated/intel_acm.h b/common/generated/intel_acm.h new file mode 100644 index 0000000..43219eb --- /dev/null +++ b/common/generated/intel_acm.h @@ -0,0 +1,159 @@ +#ifndef INTEL_ACM_H_ +#define INTEL_ACM_H_ + +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "../kaitai/kaitaistruct.h" +#include + +#if KAITAI_STRUCT_VERSION < 9000L +#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required" +#endif + +class intel_acm_t : public kaitai::kstruct { + +public: + class header_t; + + enum module_subtype_t { + MODULE_SUBTYPE_TXT = 0, + MODULE_SUBTYPE_STARTUP = 1, + MODULE_SUBTYPE_BOOT_GUARD = 3 + }; + + enum known_header_version_t { + KNOWN_HEADER_VERSION_V0_0 = 0, + KNOWN_HEADER_VERSION_V3_0 = 196608 + }; + + intel_acm_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = 0, intel_acm_t* p__root = 0); + +private: + void _read(); + void _clean_up(); + +public: + ~intel_acm_t(); + + class header_t : public kaitai::kstruct { + + public: + + header_t(kaitai::kstream* p__io, intel_acm_t* p__parent = 0, intel_acm_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~header_t(); + + private: + uint16_t m_module_type; + module_subtype_t m_module_subtype; + uint32_t m_header_size; + uint32_t m_header_version; + uint16_t m_chipset_id; + uint16_t m_flags; + uint32_t m_module_vendor; + uint8_t m_date_day; + uint8_t m_date_month; + uint16_t m_date_year; + uint32_t m_module_size; + uint16_t m_acm_svn; + uint16_t m_se_svn; + uint32_t m_code_control_flags; + uint32_t m_error_entry_point; + uint32_t m_gdt_max; + uint32_t m_gdt_base; + uint32_t m_segment_sel; + uint32_t m_entry_point; + std::string m_reserved; + uint32_t m_key_size; + uint32_t m_scratch_space_size; + std::string m_rsa_public_key; + uint32_t m_rsa_exponent; + bool n_rsa_exponent; + + public: + bool _is_null_rsa_exponent() { rsa_exponent(); return n_rsa_exponent; }; + + private: + std::string m_rsa_signature; + std::string m_scratch_space; + intel_acm_t* m__root; + intel_acm_t* m__parent; + + public: + uint16_t module_type() const { return m_module_type; } + module_subtype_t module_subtype() const { return m_module_subtype; } + + /** + * counted in 4 byte increments + */ + uint32_t header_size() const { return m_header_size; } + uint32_t header_version() const { return m_header_version; } + uint16_t chipset_id() const { return m_chipset_id; } + uint16_t flags() const { return m_flags; } + uint32_t module_vendor() const { return m_module_vendor; } + + /** + * BCD + */ + uint8_t date_day() const { return m_date_day; } + + /** + * BCD + */ + uint8_t date_month() const { return m_date_month; } + + /** + * BCD + */ + uint16_t date_year() const { return m_date_year; } + + /** + * counted in 4 byte increments + */ + uint32_t module_size() const { return m_module_size; } + uint16_t acm_svn() const { return m_acm_svn; } + uint16_t se_svn() const { return m_se_svn; } + uint32_t code_control_flags() const { return m_code_control_flags; } + uint32_t error_entry_point() const { return m_error_entry_point; } + uint32_t gdt_max() const { return m_gdt_max; } + uint32_t gdt_base() const { return m_gdt_base; } + uint32_t segment_sel() const { return m_segment_sel; } + uint32_t entry_point() const { return m_entry_point; } + std::string reserved() const { return m_reserved; } + + /** + * counted in 4 byte increments + */ + uint32_t key_size() const { return m_key_size; } + + /** + * counted in 4 byte increments + */ + uint32_t scratch_space_size() const { return m_scratch_space_size; } + std::string rsa_public_key() const { return m_rsa_public_key; } + uint32_t rsa_exponent() const { return m_rsa_exponent; } + std::string rsa_signature() const { return m_rsa_signature; } + std::string scratch_space() const { return m_scratch_space; } + intel_acm_t* _root() const { return m__root; } + intel_acm_t* _parent() const { return m__parent; } + }; + +private: + header_t* m_header; + std::string m_body; + intel_acm_t* m__root; + kaitai::kstruct* m__parent; + +public: + header_t* header() const { return m_header; } + std::string body() const { return m_body; } + intel_acm_t* _root() const { return m__root; } + kaitai::kstruct* _parent() const { return m__parent; } +}; + +#endif // INTEL_ACM_H_ diff --git a/common/generated/intel_keym_v1.cpp b/common/generated/intel_keym_v1.cpp new file mode 100644 index 0000000..0f83d28 --- /dev/null +++ b/common/generated/intel_keym_v1.cpp @@ -0,0 +1,162 @@ +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "intel_keym_v1.h" +#include "../kaitai/exceptions.h" + +intel_keym_v1_t::intel_keym_v1_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, intel_keym_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = this; (void)p__root; + m_km_hash = 0; + m_key_signature = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_keym_v1_t::_read() { + m_structure_id = static_cast(m__io->read_u8le()); + if (!(structure_id() == intel_keym_v1_t::STRUCTURE_IDS_KEYM)) { + throw kaitai::validation_not_equal_error(intel_keym_v1_t::STRUCTURE_IDS_KEYM, structure_id(), _io(), std::string("/seq/0")); + } + m_version = m__io->read_u1(); + { + uint8_t _ = version(); + if (!(_ < 32)) { + throw kaitai::validation_expr_error(version(), _io(), std::string("/seq/1")); + } + } + m_km_version = m__io->read_u1(); + m_km_svn = m__io->read_u1(); + m_km_id = m__io->read_u1(); + m_km_hash = new km_hash_t(m__io, this, m__root); + m_key_signature = new key_signature_t(m__io, this, m__root); +} + +intel_keym_v1_t::~intel_keym_v1_t() { + _clean_up(); +} + +void intel_keym_v1_t::_clean_up() { + if (m_km_hash) { + delete m_km_hash; m_km_hash = 0; + } + if (m_key_signature) { + delete m_key_signature; m_key_signature = 0; + } +} + +intel_keym_v1_t::km_hash_t::km_hash_t(kaitai::kstream* p__io, intel_keym_v1_t* p__parent, intel_keym_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_keym_v1_t::km_hash_t::_read() { + m_hash_algorithm_id = m__io->read_u2le(); + m_len_hash = m__io->read_u2le(); + m_hash = m__io->read_bytes(len_hash()); +} + +intel_keym_v1_t::km_hash_t::~km_hash_t() { + _clean_up(); +} + +void intel_keym_v1_t::km_hash_t::_clean_up() { +} + +intel_keym_v1_t::public_key_t::public_key_t(kaitai::kstream* p__io, intel_keym_v1_t::key_signature_t* p__parent, intel_keym_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_keym_v1_t::public_key_t::_read() { + m_version = m__io->read_u1(); + m_size_bits = m__io->read_u2le(); + m_exponent = m__io->read_u4le(); + m_modulus = m__io->read_bytes((size_bits() / 8)); +} + +intel_keym_v1_t::public_key_t::~public_key_t() { + _clean_up(); +} + +void intel_keym_v1_t::public_key_t::_clean_up() { +} + +intel_keym_v1_t::signature_t::signature_t(kaitai::kstream* p__io, intel_keym_v1_t::key_signature_t* p__parent, intel_keym_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_keym_v1_t::signature_t::_read() { + m_version = m__io->read_u1(); + m_size_bits = m__io->read_u2le(); + m_hash_algorithm_id = m__io->read_u2le(); + m_signature = m__io->read_bytes((size_bits() / 8)); +} + +intel_keym_v1_t::signature_t::~signature_t() { + _clean_up(); +} + +void intel_keym_v1_t::signature_t::_clean_up() { +} + +intel_keym_v1_t::key_signature_t::key_signature_t(kaitai::kstream* p__io, intel_keym_v1_t* p__parent, intel_keym_v1_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_public_key = 0; + m_signature = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_keym_v1_t::key_signature_t::_read() { + m_version = m__io->read_u1(); + m_key_id = m__io->read_u2le(); + m_public_key = new public_key_t(m__io, this, m__root); + m_sig_scheme = m__io->read_u2le(); + m_signature = new signature_t(m__io, this, m__root); +} + +intel_keym_v1_t::key_signature_t::~key_signature_t() { + _clean_up(); +} + +void intel_keym_v1_t::key_signature_t::_clean_up() { + if (m_public_key) { + delete m_public_key; m_public_key = 0; + } + if (m_signature) { + delete m_signature; m_signature = 0; + } +} diff --git a/common/generated/intel_keym_v1.h b/common/generated/intel_keym_v1.h new file mode 100644 index 0000000..adf8356 --- /dev/null +++ b/common/generated/intel_keym_v1.h @@ -0,0 +1,177 @@ +#ifndef INTEL_KEYM_V1_H_ +#define INTEL_KEYM_V1_H_ + +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "../kaitai/kaitaistruct.h" +#include + +#if KAITAI_STRUCT_VERSION < 9000L +#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required" +#endif + +class intel_keym_v1_t : public kaitai::kstruct { + +public: + class km_hash_t; + class public_key_t; + class signature_t; + class key_signature_t; + + enum structure_ids_t { + STRUCTURE_IDS_KEYM = 6872296602200661855LL + }; + + intel_keym_v1_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = 0, intel_keym_v1_t* p__root = 0); + +private: + void _read(); + void _clean_up(); + +public: + ~intel_keym_v1_t(); + + class km_hash_t : public kaitai::kstruct { + + public: + + km_hash_t(kaitai::kstream* p__io, intel_keym_v1_t* p__parent = 0, intel_keym_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~km_hash_t(); + + private: + uint16_t m_hash_algorithm_id; + uint16_t m_len_hash; + std::string m_hash; + intel_keym_v1_t* m__root; + intel_keym_v1_t* m__parent; + + public: + uint16_t hash_algorithm_id() const { return m_hash_algorithm_id; } + uint16_t len_hash() const { return m_len_hash; } + std::string hash() const { return m_hash; } + intel_keym_v1_t* _root() const { return m__root; } + intel_keym_v1_t* _parent() const { return m__parent; } + }; + + class public_key_t : public kaitai::kstruct { + + public: + + public_key_t(kaitai::kstream* p__io, intel_keym_v1_t::key_signature_t* p__parent = 0, intel_keym_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~public_key_t(); + + private: + uint8_t m_version; + uint16_t m_size_bits; + uint32_t m_exponent; + std::string m_modulus; + intel_keym_v1_t* m__root; + intel_keym_v1_t::key_signature_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t size_bits() const { return m_size_bits; } + uint32_t exponent() const { return m_exponent; } + std::string modulus() const { return m_modulus; } + intel_keym_v1_t* _root() const { return m__root; } + intel_keym_v1_t::key_signature_t* _parent() const { return m__parent; } + }; + + class signature_t : public kaitai::kstruct { + + public: + + signature_t(kaitai::kstream* p__io, intel_keym_v1_t::key_signature_t* p__parent = 0, intel_keym_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~signature_t(); + + private: + uint8_t m_version; + uint16_t m_size_bits; + uint16_t m_hash_algorithm_id; + std::string m_signature; + intel_keym_v1_t* m__root; + intel_keym_v1_t::key_signature_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t size_bits() const { return m_size_bits; } + uint16_t hash_algorithm_id() const { return m_hash_algorithm_id; } + std::string signature() const { return m_signature; } + intel_keym_v1_t* _root() const { return m__root; } + intel_keym_v1_t::key_signature_t* _parent() const { return m__parent; } + }; + + class key_signature_t : public kaitai::kstruct { + + public: + + key_signature_t(kaitai::kstream* p__io, intel_keym_v1_t* p__parent = 0, intel_keym_v1_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~key_signature_t(); + + private: + uint8_t m_version; + uint16_t m_key_id; + public_key_t* m_public_key; + uint16_t m_sig_scheme; + signature_t* m_signature; + intel_keym_v1_t* m__root; + intel_keym_v1_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t key_id() const { return m_key_id; } + public_key_t* public_key() const { return m_public_key; } + uint16_t sig_scheme() const { return m_sig_scheme; } + signature_t* signature() const { return m_signature; } + intel_keym_v1_t* _root() const { return m__root; } + intel_keym_v1_t* _parent() const { return m__parent; } + }; + +private: + structure_ids_t m_structure_id; + uint8_t m_version; + uint8_t m_km_version; + uint8_t m_km_svn; + uint8_t m_km_id; + km_hash_t* m_km_hash; + key_signature_t* m_key_signature; + intel_keym_v1_t* m__root; + kaitai::kstruct* m__parent; + +public: + structure_ids_t structure_id() const { return m_structure_id; } + uint8_t version() const { return m_version; } + uint8_t km_version() const { return m_km_version; } + uint8_t km_svn() const { return m_km_svn; } + uint8_t km_id() const { return m_km_id; } + km_hash_t* km_hash() const { return m_km_hash; } + key_signature_t* key_signature() const { return m_key_signature; } + intel_keym_v1_t* _root() const { return m__root; } + kaitai::kstruct* _parent() const { return m__parent; } +}; + +#endif // INTEL_KEYM_V1_H_ diff --git a/common/generated/intel_keym_v2.cpp b/common/generated/intel_keym_v2.cpp new file mode 100644 index 0000000..1778a7c --- /dev/null +++ b/common/generated/intel_keym_v2.cpp @@ -0,0 +1,214 @@ +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "intel_keym_v2.h" +#include "../kaitai/exceptions.h" + +intel_keym_v2_t::intel_keym_v2_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, intel_keym_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = this; (void)p__root; + m_header = 0; + m_reserved = 0; + m_km_hashes = 0; + m_key_signature = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_keym_v2_t::_read() { + m_header = new header_t(m__io, this, m__root); + m_key_signature_offset = m__io->read_u2le(); + m_reserved = new std::vector(); + const int l_reserved = 3; + for (int i = 0; i < l_reserved; i++) { + m_reserved->push_back(m__io->read_u1()); + } + m_km_version = m__io->read_u1(); + m_km_svn = m__io->read_u1(); + m_km_id = m__io->read_u1(); + m_fpf_hash_algorithm_id = m__io->read_u2le(); + m_num_km_hashes = m__io->read_u2le(); + m_km_hashes = new std::vector(); + const int l_km_hashes = num_km_hashes(); + for (int i = 0; i < l_km_hashes; i++) { + m_km_hashes->push_back(new km_hash_t(m__io, this, m__root)); + } + m_key_signature = new key_signature_t(m__io, this, m__root); +} + +intel_keym_v2_t::~intel_keym_v2_t() { + _clean_up(); +} + +void intel_keym_v2_t::_clean_up() { + if (m_header) { + delete m_header; m_header = 0; + } + if (m_reserved) { + delete m_reserved; m_reserved = 0; + } + if (m_km_hashes) { + for (std::vector::iterator it = m_km_hashes->begin(); it != m_km_hashes->end(); ++it) { + delete *it; + } + delete m_km_hashes; m_km_hashes = 0; + } + if (m_key_signature) { + delete m_key_signature; m_key_signature = 0; + } +} + +intel_keym_v2_t::key_signature_t::key_signature_t(kaitai::kstream* p__io, intel_keym_v2_t* p__parent, intel_keym_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + m_public_key = 0; + m_signature = 0; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_keym_v2_t::key_signature_t::_read() { + m_version = m__io->read_u1(); + m_key_id = m__io->read_u2le(); + m_public_key = new public_key_t(m__io, this, m__root); + m_sig_scheme = m__io->read_u2le(); + m_signature = new signature_t(m__io, this, m__root); +} + +intel_keym_v2_t::key_signature_t::~key_signature_t() { + _clean_up(); +} + +void intel_keym_v2_t::key_signature_t::_clean_up() { + if (m_public_key) { + delete m_public_key; m_public_key = 0; + } + if (m_signature) { + delete m_signature; m_signature = 0; + } +} + +intel_keym_v2_t::km_hash_t::km_hash_t(kaitai::kstream* p__io, intel_keym_v2_t* p__parent, intel_keym_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_keym_v2_t::km_hash_t::_read() { + m_usage_flags = m__io->read_u8le(); + m_hash_algorithm_id = m__io->read_u2le(); + m_len_hash = m__io->read_u2le(); + m_hash = m__io->read_bytes(len_hash()); +} + +intel_keym_v2_t::km_hash_t::~km_hash_t() { + _clean_up(); +} + +void intel_keym_v2_t::km_hash_t::_clean_up() { +} + +intel_keym_v2_t::signature_t::signature_t(kaitai::kstream* p__io, intel_keym_v2_t::key_signature_t* p__parent, intel_keym_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_keym_v2_t::signature_t::_read() { + m_version = m__io->read_u1(); + m_size_bits = m__io->read_u2le(); + m_hash_algorithm_id = m__io->read_u2le(); + m_signature = m__io->read_bytes((size_bits() / 8)); +} + +intel_keym_v2_t::signature_t::~signature_t() { + _clean_up(); +} + +void intel_keym_v2_t::signature_t::_clean_up() { +} + +intel_keym_v2_t::public_key_t::public_key_t(kaitai::kstream* p__io, intel_keym_v2_t::key_signature_t* p__parent, intel_keym_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_keym_v2_t::public_key_t::_read() { + m_version = m__io->read_u1(); + m_size_bits = m__io->read_u2le(); + m_exponent = m__io->read_u4le(); + m_modulus = m__io->read_bytes((size_bits() / 8)); +} + +intel_keym_v2_t::public_key_t::~public_key_t() { + _clean_up(); +} + +void intel_keym_v2_t::public_key_t::_clean_up() { +} + +intel_keym_v2_t::header_t::header_t(kaitai::kstream* p__io, intel_keym_v2_t* p__parent, intel_keym_v2_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void intel_keym_v2_t::header_t::_read() { + m_structure_id = static_cast(m__io->read_u8le()); + if (!(structure_id() == intel_keym_v2_t::STRUCTURE_IDS_KEYM)) { + throw kaitai::validation_not_equal_error(intel_keym_v2_t::STRUCTURE_IDS_KEYM, structure_id(), _io(), std::string("/types/header/seq/0")); + } + m_version = m__io->read_u1(); + { + uint8_t _ = version(); + if (!(_ >= 32)) { + throw kaitai::validation_expr_error(version(), _io(), std::string("/types/header/seq/1")); + } + } + m_header_specific = m__io->read_u1(); + m_total_size = m__io->read_u2le(); + if (!(total_size() == 0)) { + throw kaitai::validation_not_equal_error(0, total_size(), _io(), std::string("/types/header/seq/3")); + } +} + +intel_keym_v2_t::header_t::~header_t() { + _clean_up(); +} + +void intel_keym_v2_t::header_t::_clean_up() { +} diff --git a/common/generated/intel_keym_v2.h b/common/generated/intel_keym_v2.h new file mode 100644 index 0000000..062228f --- /dev/null +++ b/common/generated/intel_keym_v2.h @@ -0,0 +1,224 @@ +#ifndef INTEL_KEYM_V2_H_ +#define INTEL_KEYM_V2_H_ + +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "../kaitai/kaitaistruct.h" +#include +#include + +#if KAITAI_STRUCT_VERSION < 9000L +#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required" +#endif + +class intel_keym_v2_t : public kaitai::kstruct { + +public: + class key_signature_t; + class km_hash_t; + class signature_t; + class public_key_t; + class header_t; + + enum structure_ids_t { + STRUCTURE_IDS_KEYM = 6872296602200661855LL + }; + + enum km_usage_flags_t { + KM_USAGE_FLAGS_BOOT_POLICY_MANIFEST = 1, + KM_USAGE_FLAGS_FIT_PATCH_MANIFEST = 2, + KM_USAGE_FLAGS_ACM_MANIFEST = 4, + KM_USAGE_FLAGS_SDEV = 8 + }; + + intel_keym_v2_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = 0, intel_keym_v2_t* p__root = 0); + +private: + void _read(); + void _clean_up(); + +public: + ~intel_keym_v2_t(); + + class key_signature_t : public kaitai::kstruct { + + public: + + key_signature_t(kaitai::kstream* p__io, intel_keym_v2_t* p__parent = 0, intel_keym_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~key_signature_t(); + + private: + uint8_t m_version; + uint16_t m_key_id; + public_key_t* m_public_key; + uint16_t m_sig_scheme; + signature_t* m_signature; + intel_keym_v2_t* m__root; + intel_keym_v2_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t key_id() const { return m_key_id; } + public_key_t* public_key() const { return m_public_key; } + uint16_t sig_scheme() const { return m_sig_scheme; } + signature_t* signature() const { return m_signature; } + intel_keym_v2_t* _root() const { return m__root; } + intel_keym_v2_t* _parent() const { return m__parent; } + }; + + class km_hash_t : public kaitai::kstruct { + + public: + + km_hash_t(kaitai::kstream* p__io, intel_keym_v2_t* p__parent = 0, intel_keym_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~km_hash_t(); + + private: + uint64_t m_usage_flags; + uint16_t m_hash_algorithm_id; + uint16_t m_len_hash; + std::string m_hash; + intel_keym_v2_t* m__root; + intel_keym_v2_t* m__parent; + + public: + uint64_t usage_flags() const { return m_usage_flags; } + uint16_t hash_algorithm_id() const { return m_hash_algorithm_id; } + uint16_t len_hash() const { return m_len_hash; } + std::string hash() const { return m_hash; } + intel_keym_v2_t* _root() const { return m__root; } + intel_keym_v2_t* _parent() const { return m__parent; } + }; + + class signature_t : public kaitai::kstruct { + + public: + + signature_t(kaitai::kstream* p__io, intel_keym_v2_t::key_signature_t* p__parent = 0, intel_keym_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~signature_t(); + + private: + uint8_t m_version; + uint16_t m_size_bits; + uint16_t m_hash_algorithm_id; + std::string m_signature; + intel_keym_v2_t* m__root; + intel_keym_v2_t::key_signature_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t size_bits() const { return m_size_bits; } + uint16_t hash_algorithm_id() const { return m_hash_algorithm_id; } + std::string signature() const { return m_signature; } + intel_keym_v2_t* _root() const { return m__root; } + intel_keym_v2_t::key_signature_t* _parent() const { return m__parent; } + }; + + class public_key_t : public kaitai::kstruct { + + public: + + public_key_t(kaitai::kstream* p__io, intel_keym_v2_t::key_signature_t* p__parent = 0, intel_keym_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~public_key_t(); + + private: + uint8_t m_version; + uint16_t m_size_bits; + uint32_t m_exponent; + std::string m_modulus; + intel_keym_v2_t* m__root; + intel_keym_v2_t::key_signature_t* m__parent; + + public: + uint8_t version() const { return m_version; } + uint16_t size_bits() const { return m_size_bits; } + uint32_t exponent() const { return m_exponent; } + std::string modulus() const { return m_modulus; } + intel_keym_v2_t* _root() const { return m__root; } + intel_keym_v2_t::key_signature_t* _parent() const { return m__parent; } + }; + + class header_t : public kaitai::kstruct { + + public: + + header_t(kaitai::kstream* p__io, intel_keym_v2_t* p__parent = 0, intel_keym_v2_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~header_t(); + + private: + structure_ids_t m_structure_id; + uint8_t m_version; + uint8_t m_header_specific; + uint16_t m_total_size; + intel_keym_v2_t* m__root; + intel_keym_v2_t* m__parent; + + public: + structure_ids_t structure_id() const { return m_structure_id; } + uint8_t version() const { return m_version; } + uint8_t header_specific() const { return m_header_specific; } + uint16_t total_size() const { return m_total_size; } + intel_keym_v2_t* _root() const { return m__root; } + intel_keym_v2_t* _parent() const { return m__parent; } + }; + +private: + header_t* m_header; + uint16_t m_key_signature_offset; + std::vector* m_reserved; + uint8_t m_km_version; + uint8_t m_km_svn; + uint8_t m_km_id; + uint16_t m_fpf_hash_algorithm_id; + uint16_t m_num_km_hashes; + std::vector* m_km_hashes; + key_signature_t* m_key_signature; + intel_keym_v2_t* m__root; + kaitai::kstruct* m__parent; + +public: + header_t* header() const { return m_header; } + uint16_t key_signature_offset() const { return m_key_signature_offset; } + std::vector* reserved() const { return m_reserved; } + uint8_t km_version() const { return m_km_version; } + uint8_t km_svn() const { return m_km_svn; } + uint8_t km_id() const { return m_km_id; } + uint16_t fpf_hash_algorithm_id() const { return m_fpf_hash_algorithm_id; } + uint16_t num_km_hashes() const { return m_num_km_hashes; } + std::vector* km_hashes() const { return m_km_hashes; } + key_signature_t* key_signature() const { return m_key_signature; } + intel_keym_v2_t* _root() const { return m__root; } + kaitai::kstruct* _parent() const { return m__parent; } +}; + +#endif // INTEL_KEYM_V2_H_ diff --git a/common/intel_fit.h b/common/intel_fit.h new file mode 100755 index 0000000..86d1604 --- /dev/null +++ b/common/intel_fit.h @@ -0,0 +1,226 @@ +/* intel_fit.h + +Copyright (c) 2015, Nikolaj Schlej. All rights reserved. +This program and the accompanying materials +are licensed and made available under the terms and conditions of the BSD License +which accompanies this distribution. The full text of the license may be found at +http://opensource.org/licenses/bsd-license.php + +THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, +WITHWARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. +*/ + +#ifndef INTEL_FIT_H +#define INTEL_FIT_H + +#include "basetypes.h" +#include "ubytearray.h" + +// Make sure we use right packing rules +#pragma pack(push, 1) + +// Memory address of a pointer to FIT, 40h back from the end of flash chip +#define INTEL_FIT_POINTER_OFFSET 0x40 + +// Entry types +// https://www.intel.com/content/dam/develop/external/us/en/documents/firmware-interface-table-bios-specification-r1p2p1.pdf +#define INTEL_FIT_TYPE_HEADER 0x00 +#define INTEL_FIT_TYPE_MICROCODE 0x01 +#define INTEL_FIT_TYPE_STARTUP_AC_MODULE 0x02 +#define INTEL_FIT_TYPE_DIAG_AC_MODULE 0x03 +//#define INTEL_FIT_TYPE_INTEL_RESERVED_04 0x04 +//#define INTEL_FIT_TYPE_INTEL_RESERVED_05 0x05 +//#define INTEL_FIT_TYPE_INTEL_RESERVED_06 0x06 +#define INTEL_FIT_TYPE_BIOS_STARTUP_MODULE 0x07 +#define INTEL_FIT_TYPE_TPM_POLICY 0x08 +#define INTEL_FIT_TYPE_BIOS_POLICY 0x09 +#define INTEL_FIT_TYPE_TXT_POLICY 0x0A +#define INTEL_FIT_TYPE_BOOT_GUARD_KEY_MANIFEST 0x0B +#define INTEL_FIT_TYPE_BOOT_GUARD_BOOT_POLICY 0x0C +//#define INTEL_FIT_TYPE_INTEL_RESERVED_0D 0x0D +//#define INTEL_FIT_TYPE_INTEL_RESERVED_0E 0x0E +//#define INTEL_FIT_TYPE_INTEL_RESERVED_0F 0x0F +#define INTEL_FIT_TYPE_CSE_SECURE_BOOT 0x10 +//#define INTEL_FIT_TYPE_INTEL_RESERVED_11 0x11 +//... +//#define INTEL_FIT_TYPE_INTEL_RESERVED_2C 0x2C +#define INTEL_FIT_TYPE_ACM_FEATURE_POLICY 0x2D +//#define INTEL_FIT_TYPE_INTEL_RESERVED_2E 0x2E +#define INTEL_FIT_TYPE_JMP_DEBUG_POLICY 0x2F +#define INTEL_FIT_TYPE_OEM_RESERVED_30 0x30 +//... +#define INTEL_FIT_TYPE_OEM_RESERVED_70 0x70 +//#define INTEL_FIT_TYPE_INTEL_RESERVED_71 0x71 +//... +//#define INTEL_FIT_TYPE_INTEL_RESERVED_7E 0x7E +#define INTEL_FIT_TYPE_EMPTY 0x7F + +typedef struct INTEL_FIT_ENTRY_ { + UINT64 Address; // Base address of the component, must be 16-byte aligned + UINT32 Size : 24; // Size of the component, in multiple of 16 bytes + UINT32 Reserved : 8; // Reserved, must be set to zero + UINT16 Version; // BCD, minor in lower byte, major in upper byte + UINT8 Type : 7; // FIT entries must be aranged in ascending order of Type + UINT8 ChecksumValid : 1;// Checksum must be ignored if this bit is not set + UINT8 Checksum; // checksum8 of all the bytes in the component and this field must add to zero +} INTEL_FIT_ENTRY; + +// +// FIT Header +// +// Can be exactly one entry of this type, the first one. +// If ChecksumValid bit is set, the whole FIT table must checksum8 to zero. +#define INTEL_FIT_SIGNATURE 0x2020205F5449465FULL // '_FIT_ ' +#define INTEL_FIT_HEADER_VERSION 0x0100 + +// +// Microcode +// +// At least one entry is required, more is optional +// Each entry must point to a valid base address +// Microcode slots can be empty (first 4 bytes at the base address are FF FF FF FF) +// Base address must be aligned to 16 bytes +// The item at the base address must not be compressed/encoded/encrypted +// ChecksumValid bit must be 0 +// Size is not used, should be set to 0 + +// +// Startup Authenticated Code Module +// +// Optional, required for AC boot and BootGuard +// Address must point to a valid base address +// Points to the first byte of ACM header +// One MTRR base/limit pair is used to map Startup ACM, so +// MTRR_Base must be a multiple of MTRR_Size, the can be found by the following formula +// MTTR_Size = 2^(ceil(log2(Startup_ACM_Size))), i.e. the next integer that's a full power of 2 to Startup_ACM_Size +// The whole area of [MTRR_Base; MTRR_Base + MTRR_Size) is named +// Authenticated Code Execution Area (ACEA) and should not contain any code or data that is not the Startup ACM itself +// ChecksumValid bit must be 0 +// Size is not used, should be set to 0 +#define INTEL_FIT_STARTUP_ACM_VERSION 0x0100 +#define INTEL_ACM_HARDCODED_RSA_EXPONENT 0x10001 + +// +// Diagnostic Authenticated Code Module +// +// Optional +// Address must point to a valid base address +// Points to the first byte of ACM header, that must be 4Kb-aligned +// ChecksumValid bit must be 0 +// Size is not used, should be set to 0 +#define INTEL_FIT_DIAG_ACM_VERSION 0x0100 + +// +// BIOS Startup Module +// +// Optional, used for legacy TXT FIT boot +// Address must point to a valid base address +// At least one entry of this type must point to an item containing the reset vector +// At least one entry of this type must point to an item containing the FIT pointer +// No entries of this type can point to an item containing an item pointed by INTEL_FIT_TYPE_BIOS_POLICY entry +// Items pointed by entries of this type can not overlap with one another or the Startup ACM +// ChecksumValid bit must be 0 +// Size is used and in 16-byte multiples +#define INTEL_FIT_BIOS_STARTUP_MODULE_VERSION 0x0100 + +// +// TPM Boot Policy +// +// Optional, used for legacy TXT FIT boot, if used, can be only one +// Address entry is INTEL_FIT_POLICY_PTR.IndexIo if Version is 0, +// or INTEL_FIT_INDEX_IO_ADDRESS.FlatMemoryAddress if Version is 1 +// Bit 0 at the pointed address holds the TPM policy, 0 - TPM disabled, 1 - TPM enabled +// ChecksumValid bit must be 0 +// Size is not used, should be set to 0 +typedef struct INTEL_FIT_INDEX_IO_ADDRESS_ { + UINT16 IndexRegisterAddress; + UINT16 DataRegisterAddress; + UINT8 AccessWidthInBytes; // 1 => 1-byte accesses, 2 => 2-byte + UINT8 BitPosition; // Bit number, 15 => Bit15 + UINT16 Index; +} INTEL_FIT_INDEX_IO_ADDRESS; + +typedef union INTEL_FIT_POLICY_PTR_ { + UINT64 FlatMemoryAddress; + INTEL_FIT_INDEX_IO_ADDRESS IndexIo; +} INTEL_FIT_POLICY_PTR; + +#define INTEL_FIT_POLICY_VERSION_INDEX_IO 0 +#define INTEL_FIT_POLICY_VERSION_FLAT_MEMORY_ADDRESS 1 + +#define INTEL_FIT_POLICY_DISABLED 1 +#define INTEL_FIT_POLICY_ENABLED 1 + +// +// BIOS Policy +// +// Optional, used for legacy TXT FIT boot, if used, can be only one +// Address must point to a valid base address +// Points to the first byte of LCP_POLICY_DATA structure +// ChecksumValid bit must be 0 +// Size must not be less than the size of LCP_POLICY_DATA structure +#define INTEL_FIT_BIOS_POLICY_VERSION 0x0100 + + +// +// TXT Boot Policy +// +// Optional, if used, can be only one +// Address entry is INTEL_FIT_POLICY_PTR.IndexIo if Version is 0, +// or INTEL_FIT_INDEX_IO_ADDRESS.FlatMemoryAddress if Version is 1 +// Bit 0 at the pointed address holds the TXT policy, 0 - TXT disabled, 1 - TXT enabled +// ChecksumValid bit must be 0 +// Size is not used, should be set to 0 + +// +// BootGuard Key Manifest +// +// Optional, can be multiple, entries must be grouped together +// Address must point to a valid base address +// ChecksumValid bit must be 0 +// Size must not be less than the size of INTEL_BOOT_GUARD_KEY_MANIFEST structure +#define INTEL_FIT_BOOT_GUARD_KEY_MANIFEST_VERSION 0x0100 + +// +// BootGuard Boot Policy Manifest +// +// Optional, can be multiple, only the first one will be used +// Address must point to a valid base address +// ChecksumValid bit must be 0 +// Size must not be less than the size of INTEL_BOOT_GUARD_BOOT_POLICY structure +#define INTEL_FIT_BOOT_GUARD_KEY_MANIFEST_VERSION 0x0100 + +// +// CSE SecureBoot +// +// Optional, can be multiple, order is not important +// If present, BootGuardKeyManifest and BootGuardBootPolicy should also be present +// Reserved field further dermines the subtype of this entry +// ChecksumValid bit must be 0 +#define INTEL_FIT_CSE_SECURE_BOOT_VERSION 0x0100 + +#define INTEL_FIT_CSE_SECURE_BOOT_RESERVED 0 +#define INTEL_FIT_CSE_SECURE_BOOT_KEY_HASH 1 +#define INTEL_FIT_CSE_SECURE_BOOT_CSE_MEASUREMENT_HASH 2 +#define INTEL_FIT_CSE_SECURE_BOOT_BOOT_POLICY 3 +#define INTEL_FIT_CSE_SECURE_BOOT_OTHER_BOOT_POLICY 4 +#define INTEL_FIT_CSE_SECURE_BOOT_OEM_SMIP 5 +#define INTEL_FIT_CSE_SECURE_BOOT_MRC_TRAINING_DATA 6 +#define INTEL_FIT_CSE_SECURE_BOOT_IBBL_HASH 7 +#define INTEL_FIT_CSE_SECURE_BOOT_IBB_HASH 8 +#define INTEL_FIT_CSE_SECURE_BOOT_OEM_ID 9 +#define INTEL_FIT_CSE_SECURE_BOOT_OEM_SKU_ID 10 +#define INTEL_FIT_CSE_SECURE_BOOT_BOOT_DEVICE_INDICATOR 11 // 1 => SPI, 2 => eMMC, 3 => UFS, rest => reserved +#define INTEL_FIT_CSE_SECURE_BOOT_FIT_PATCH_MANIFEST 12 +#define INTEL_FIT_CSE_SECURE_BOOT_AC_MODULE_MANIFEST 13 + +// +// ACM Feature Policy Record +// +// Optional, can be multiple +// ChecksumValid bit must be 0 +#define INTEL_FIT_ACM_FEATURE_POLICY_VERSION 0x0100 + +#pragma pack(pop) + +#endif // INTEL_FIT_H diff --git a/common/fit.h b/common/intel_microcode.h old mode 100755 new mode 100644 similarity index 72% rename from common/fit.h rename to common/intel_microcode.h index 18992a3..c6a3d89 --- a/common/fit.h +++ b/common/intel_microcode.h @@ -1,6 +1,6 @@ -/* fit.h +/* intel_microcode.h -Copyright (c) 2015, Nikolaj Schlej. All rights reserved. +Copyright (c) 2022, Nikolaj Schlej. All rights reserved. This program and the accompanying materials are licensed and made available under the terms and conditions of the BSD License which accompanies this distribution. The full text of the license may be found at @@ -10,8 +10,8 @@ THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, WITHWARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. */ -#ifndef FIT_H -#define FIT_H +#ifndef INTEL_MICROCODE_H +#define INTEL_MICROCODE_H #include "basetypes.h" #include "ubytearray.h" @@ -19,46 +19,7 @@ WITHWARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. // Make sure we use right packing rules #pragma pack(push, 1) -// Memory address of a pointer to FIT, 40h back from the end of flash chip -#define FIT_POINTER_OFFSET 0x40 - -// Entry types -#define FIT_TYPE_HEADER 0x00 -#define FIT_TYPE_MICROCODE 0x01 -#define FIT_TYPE_BIOS_AC_MODULE 0x02 -#define FIT_TYPE_BIOS_INIT_MODULE 0x07 -#define FIT_TYPE_TPM_POLICY 0x08 -#define FIT_TYPE_BIOS_POLICY_DATA 0x09 -#define FIT_TYPE_TXT_CONF_POLICY 0x0A -#define FIT_TYPE_AC_KEY_MANIFEST 0x0B -#define FIT_TYPE_AC_BOOT_POLICY 0x0C -#define FIT_TYPE_EMPTY 0x7F - -#define FIT_HEADER_VERSION 0x0100 -#define FIT_MICROCODE_VERSION 0x0100 - -const UByteArray FIT_SIGNATURE -("\x5F\x46\x49\x54\x5F\x20\x20\x20", 8); - -typedef struct FIT_ENTRY_ { - UINT64 Address; - UINT32 Size : 24; - UINT32 : 8; - UINT16 Version; - UINT8 Type : 7; - UINT8 CsFlag : 1; - UINT8 Checksum; -} FIT_ENTRY; - -typedef struct { - UINT16 IndexRegisterAddress; - UINT16 DataRegisterAddress; - UINT8 AccessWidth; - UINT8 BitPosition; - UINT16 Index; -} FIT_ENTRY_VERSION_0_CONFIG_POLICY; - -// This scructure is described in Section 9.11.1 of the Intel Software Developer manual Volume 3A Part 1 +// This structure is described in Section 9.11.1 of the Intel Software Developer manual Volume 3A Part 1 typedef struct INTEL_MICROCODE_HEADER_ { UINT32 HeaderVersion; // 0x00000001 UINT32 UpdateRevision; @@ -102,8 +63,8 @@ typedef struct INTEL_MICROCODE_EXTENDED_HEADER_ENTRY_ { // Checksum is correct when the summation of all DWORDs that comprise the created Extended Processor Patch results in 00000000H. } INTEL_MICROCODE_EXTENDED_HEADER_ENTRY; -#define INTEL_MICROCODE_HEADER_VERSION_1 0x00000001 +#define INTEL_MICROCODE_HEADER_VERSION_1 0x00000001 #pragma pack(pop) -#endif // FIT_H +#endif // INTEL_MICROCODE_H diff --git a/common/kaitai/custom_decoder.h b/common/kaitai/custom_decoder.h new file mode 100644 index 0000000..6da7f5f --- /dev/null +++ b/common/kaitai/custom_decoder.h @@ -0,0 +1,16 @@ +#ifndef KAITAI_CUSTOM_DECODER_H +#define KAITAI_CUSTOM_DECODER_H + +#include + +namespace kaitai { + +class custom_decoder { +public: + virtual ~custom_decoder() {}; + virtual std::string decode(std::string src) = 0; +}; + +} + +#endif diff --git a/common/kaitai/exceptions.h b/common/kaitai/exceptions.h new file mode 100644 index 0000000..bb414dc --- /dev/null +++ b/common/kaitai/exceptions.h @@ -0,0 +1,189 @@ +#ifndef KAITAI_EXCEPTIONS_H +#define KAITAI_EXCEPTIONS_H + +#include "kaitaistream.h" + +#include +#include + +// We need to use "noexcept" in virtual destructor of our exceptions +// subclasses. Different compilers have different ideas on how to +// achieve that: C++98 compilers prefer `throw()`, C++11 and later +// use `noexcept`. We define KS_NOEXCEPT macro for that. + +#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1900) +#define KS_NOEXCEPT noexcept +#else +#define KS_NOEXCEPT throw() +#endif + +namespace kaitai { + +/** + * Common ancestor for all error originating from Kaitai Struct usage. + * Stores KSY source path, pointing to an element supposedly guilty of + * an error. + */ +class kstruct_error: public std::runtime_error { +public: + kstruct_error(const std::string what, const std::string src_path): + std::runtime_error(src_path + ": " + what), + m_src_path(src_path) + { + } + + virtual ~kstruct_error() KS_NOEXCEPT {}; + +protected: + const std::string m_src_path; +}; + +/** + * Error that occurs when default endianness should be decided with + * a switch, but nothing matches (although using endianness expression + * implies that there should be some positive result). + */ +class undecided_endianness_error: public kstruct_error { +public: + undecided_endianness_error(const std::string src_path): + kstruct_error("unable to decide on endianness for a type", src_path) + { + } + + virtual ~undecided_endianness_error() KS_NOEXCEPT {}; +}; + +/** + * Common ancestor for all validation failures. Stores pointer to + * KaitaiStream IO object which was involved in an error. + */ +class validation_failed_error: public kstruct_error { +public: + validation_failed_error(const std::string what, kstream* io, const std::string src_path): + kstruct_error("at pos " + kstream::to_string(io->pos()) + ": validation failed: " + what, src_path), + m_io(io) + { + } + +// "at pos #{io.pos}: validation failed: #{msg}" + + virtual ~validation_failed_error() KS_NOEXCEPT {}; + +protected: + kstream* m_io; +}; + +/** + * Signals validation failure: we required "actual" value to be equal to + * "expected", but it turned out that it's not. + */ +template +class validation_not_equal_error: public validation_failed_error { +public: + validation_not_equal_error(const T& expected, const T& actual, kstream* io, const std::string src_path): + validation_failed_error("not equal", io, src_path), + m_expected(expected), + m_actual(actual) + { + } + + // "not equal, expected #{expected.inspect}, but got #{actual.inspect}" + + virtual ~validation_not_equal_error() KS_NOEXCEPT {}; + +protected: + const T& m_expected; + const T& m_actual; +}; + +/** + * Signals validation failure: we required "actual" value to be greater + * than or equal to "min", but it turned out that it's not. + */ +template +class validation_less_than_error: public validation_failed_error { +public: + validation_less_than_error(const T& min, const T& actual, kstream* io, const std::string src_path): + validation_failed_error("not in range", io, src_path), + m_min(min), + m_actual(actual) + { + } + + // "not in range, min #{min.inspect}, but got #{actual.inspect}" + + virtual ~validation_less_than_error() KS_NOEXCEPT {}; + +protected: + const T& m_min; + const T& m_actual; +}; + +/** + * Signals validation failure: we required "actual" value to be less + * than or equal to "max", but it turned out that it's not. + */ +template +class validation_greater_than_error: public validation_failed_error { +public: + validation_greater_than_error(const T& max, const T& actual, kstream* io, const std::string src_path): + validation_failed_error("not in range", io, src_path), + m_max(max), + m_actual(actual) + { + } + + // "not in range, max #{max.inspect}, but got #{actual.inspect}" + + virtual ~validation_greater_than_error() KS_NOEXCEPT {}; + +protected: + const T& m_max; + const T& m_actual; +}; + +/** + * Signals validation failure: we required "actual" value to be from + * the list, but it turned out that it's not. + */ +template +class validation_not_any_of_error: public validation_failed_error { +public: + validation_not_any_of_error(const T& actual, kstream* io, const std::string src_path): + validation_failed_error("not any of the list", io, src_path), + m_actual(actual) + { + } + + // "not any of the list, got #{actual.inspect}" + + virtual ~validation_not_any_of_error() KS_NOEXCEPT {}; + +protected: + const T& m_actual; +}; + +/** + * Signals validation failure: we required "actual" value to match + * the expression, but it turned out that it doesn't. + */ +template +class validation_expr_error: public validation_failed_error { +public: + validation_expr_error(const T& actual, kstream* io, const std::string src_path): + validation_failed_error("not matching the expression", io, src_path), + m_actual(actual) + { + } + + // "not matching the expression, got #{actual.inspect}" + + virtual ~validation_expr_error() KS_NOEXCEPT {}; + +protected: + const T& m_actual; +}; + +} + +#endif diff --git a/common/kaitai/kaitaistream.cpp b/common/kaitai/kaitaistream.cpp new file mode 100644 index 0000000..e756613 --- /dev/null +++ b/common/kaitai/kaitaistream.cpp @@ -0,0 +1,693 @@ +#include "kaitaistream.h" + +#define KS_STR_ENCODING_NONE + +#if defined(__APPLE__) +#include +#include +#define bswap_16(x) OSSwapInt16(x) +#define bswap_32(x) OSSwapInt32(x) +#define bswap_64(x) OSSwapInt64(x) +#define __BYTE_ORDER BYTE_ORDER +#define __BIG_ENDIAN BIG_ENDIAN +#define __LITTLE_ENDIAN LITTLE_ENDIAN +#elif defined(_MSC_VER) // !__APPLE__ +#include +#define __LITTLE_ENDIAN 1234 +#define __BIG_ENDIAN 4321 +#define __BYTE_ORDER __LITTLE_ENDIAN +#define bswap_16(x) _byteswap_ushort(x) +#define bswap_32(x) _byteswap_ulong(x) +#define bswap_64(x) _byteswap_uint64(x) +#else // !__APPLE__ or !_MSC_VER +#include +#include +#endif + +#include +#include +#include + +kaitai::kstream::kstream(std::istream *io) { + m_io = io; + init(); +} + +kaitai::kstream::kstream(const std::string &data) : m_io_str(data) { + m_io = &m_io_str; + init(); +} + +void kaitai::kstream::init() { + exceptions_enable(); + align_to_byte(); +} + +void kaitai::kstream::close() { + // m_io->close(); +} + +void kaitai::kstream::exceptions_enable() const { + m_io->exceptions( + std::istream::eofbit | + std::istream::failbit | + std::istream::badbit + ); +} + +// ======================================================================== +// Stream positioning +// ======================================================================== + +bool kaitai::kstream::is_eof() const { + if (m_bits_left > 0) { + return false; + } + char t; + m_io->exceptions(std::istream::badbit); + m_io->get(t); + if (m_io->eof()) { + m_io->clear(); + exceptions_enable(); + return true; + } else { + m_io->unget(); + exceptions_enable(); + return false; + } +} + +void kaitai::kstream::seek(uint64_t pos) { + m_io->seekg(pos); +} + +uint64_t kaitai::kstream::pos() { + return m_io->tellg(); +} + +uint64_t kaitai::kstream::size() { + std::iostream::pos_type cur_pos = m_io->tellg(); + m_io->seekg(0, std::ios::end); + std::iostream::pos_type len = m_io->tellg(); + m_io->seekg(cur_pos); + return len; +} + +// ======================================================================== +// Integer numbers +// ======================================================================== + +// ------------------------------------------------------------------------ +// Signed +// ------------------------------------------------------------------------ + +int8_t kaitai::kstream::read_s1() { + char t; + m_io->get(t); + return t; +} + +// ........................................................................ +// Big-endian +// ........................................................................ + +int16_t kaitai::kstream::read_s2be() { + int16_t t; + m_io->read(reinterpret_cast(&t), 2); +#if __BYTE_ORDER == __LITTLE_ENDIAN + t = bswap_16(t); +#endif + return t; +} + +int32_t kaitai::kstream::read_s4be() { + int32_t t; + m_io->read(reinterpret_cast(&t), 4); +#if __BYTE_ORDER == __LITTLE_ENDIAN + t = bswap_32(t); +#endif + return t; +} + +int64_t kaitai::kstream::read_s8be() { + int64_t t; + m_io->read(reinterpret_cast(&t), 8); +#if __BYTE_ORDER == __LITTLE_ENDIAN + t = bswap_64(t); +#endif + return t; +} + +// ........................................................................ +// Little-endian +// ........................................................................ + +int16_t kaitai::kstream::read_s2le() { + int16_t t; + m_io->read(reinterpret_cast(&t), 2); +#if __BYTE_ORDER == __BIG_ENDIAN + t = bswap_16(t); +#endif + return t; +} + +int32_t kaitai::kstream::read_s4le() { + int32_t t; + m_io->read(reinterpret_cast(&t), 4); +#if __BYTE_ORDER == __BIG_ENDIAN + t = bswap_32(t); +#endif + return t; +} + +int64_t kaitai::kstream::read_s8le() { + int64_t t; + m_io->read(reinterpret_cast(&t), 8); +#if __BYTE_ORDER == __BIG_ENDIAN + t = bswap_64(t); +#endif + return t; +} + +// ------------------------------------------------------------------------ +// Unsigned +// ------------------------------------------------------------------------ + +uint8_t kaitai::kstream::read_u1() { + char t; + m_io->get(t); + return t; +} + +// ........................................................................ +// Big-endian +// ........................................................................ + +uint16_t kaitai::kstream::read_u2be() { + uint16_t t; + m_io->read(reinterpret_cast(&t), 2); +#if __BYTE_ORDER == __LITTLE_ENDIAN + t = bswap_16(t); +#endif + return t; +} + +uint32_t kaitai::kstream::read_u4be() { + uint32_t t; + m_io->read(reinterpret_cast(&t), 4); +#if __BYTE_ORDER == __LITTLE_ENDIAN + t = bswap_32(t); +#endif + return t; +} + +uint64_t kaitai::kstream::read_u8be() { + uint64_t t; + m_io->read(reinterpret_cast(&t), 8); +#if __BYTE_ORDER == __LITTLE_ENDIAN + t = bswap_64(t); +#endif + return t; +} + +// ........................................................................ +// Little-endian +// ........................................................................ + +uint16_t kaitai::kstream::read_u2le() { + uint16_t t; + m_io->read(reinterpret_cast(&t), 2); +#if __BYTE_ORDER == __BIG_ENDIAN + t = bswap_16(t); +#endif + return t; +} + +uint32_t kaitai::kstream::read_u4le() { + uint32_t t; + m_io->read(reinterpret_cast(&t), 4); +#if __BYTE_ORDER == __BIG_ENDIAN + t = bswap_32(t); +#endif + return t; +} + +uint64_t kaitai::kstream::read_u8le() { + uint64_t t; + m_io->read(reinterpret_cast(&t), 8); +#if __BYTE_ORDER == __BIG_ENDIAN + t = bswap_64(t); +#endif + return t; +} + +// ======================================================================== +// Floating point numbers +// ======================================================================== + +// ........................................................................ +// Big-endian +// ........................................................................ + +float kaitai::kstream::read_f4be() { + uint32_t t; + m_io->read(reinterpret_cast(&t), 4); +#if __BYTE_ORDER == __LITTLE_ENDIAN + t = bswap_32(t); +#endif + return reinterpret_cast(t); +} + +double kaitai::kstream::read_f8be() { + uint64_t t; + m_io->read(reinterpret_cast(&t), 8); +#if __BYTE_ORDER == __LITTLE_ENDIAN + t = bswap_64(t); +#endif + return reinterpret_cast(t); +} + +// ........................................................................ +// Little-endian +// ........................................................................ + +float kaitai::kstream::read_f4le() { + uint32_t t; + m_io->read(reinterpret_cast(&t), 4); +#if __BYTE_ORDER == __BIG_ENDIAN + t = bswap_32(t); +#endif + return reinterpret_cast(t); +} + +double kaitai::kstream::read_f8le() { + uint64_t t; + m_io->read(reinterpret_cast(&t), 8); +#if __BYTE_ORDER == __BIG_ENDIAN + t = bswap_64(t); +#endif + return reinterpret_cast(t); +} + +// ======================================================================== +// Unaligned bit values +// ======================================================================== + +void kaitai::kstream::align_to_byte() { + m_bits_left = 0; + m_bits = 0; +} + +uint64_t kaitai::kstream::read_bits_int_be(int n) { + uint64_t res = 0; + + int bits_needed = n - m_bits_left; + m_bits_left = -bits_needed & 7; // `-bits_needed mod 8` + + if (bits_needed > 0) { + // 1 bit => 1 byte + // 8 bits => 1 byte + // 9 bits => 2 bytes + int bytes_needed = ((bits_needed - 1) / 8) + 1; // `ceil(bits_needed / 8)` + if (bytes_needed > 8) + throw std::runtime_error("read_bits_int_be: more than 8 bytes requested"); + uint8_t buf[8]; + m_io->read(reinterpret_cast(buf), bytes_needed); + for (int i = 0; i < bytes_needed; i++) { + res = res << 8 | buf[i]; + } + + uint64_t new_bits = res; + res = res >> m_bits_left | (bits_needed < 64 ? m_bits << bits_needed : 0); // avoid undefined behavior of `x << 64` + m_bits = new_bits; // will be masked at the end of the function + } else { + res = m_bits >> -bits_needed; // shift unneeded bits out + } + + uint64_t mask = (UINT64_C(1) << m_bits_left) - 1; // `m_bits_left` is in range 0..7, so `(1 << 64)` does not have to be considered + m_bits &= mask; + + return res; +} + +// Deprecated, use read_bits_int_be() instead. +uint64_t kaitai::kstream::read_bits_int(int n) { + return read_bits_int_be(n); +} + +uint64_t kaitai::kstream::read_bits_int_le(int n) { + uint64_t res = 0; + int bits_needed = n - m_bits_left; + + if (bits_needed > 0) { + // 1 bit => 1 byte + // 8 bits => 1 byte + // 9 bits => 2 bytes + int bytes_needed = ((bits_needed - 1) / 8) + 1; // `ceil(bits_needed / 8)` + if (bytes_needed > 8) + throw std::runtime_error("read_bits_int_le: more than 8 bytes requested"); + uint8_t buf[8]; + m_io->read(reinterpret_cast(buf), bytes_needed); + for (int i = 0; i < bytes_needed; i++) { + res |= static_cast(buf[i]) << (i * 8); + } + + // NB: for bit shift operators in C++, "if the value of the right operand is + // negative or is greater or equal to the number of bits in the promoted left + // operand, the behavior is undefined." (see + // https://en.cppreference.com/w/cpp/language/operator_arithmetic#Bitwise_shift_operators) + // So we define our desired behavior here. + uint64_t new_bits = bits_needed < 64 ? res >> bits_needed : 0; + res = res << m_bits_left | m_bits; + m_bits = new_bits; + } else { + res = m_bits; + m_bits >>= n; + } + + m_bits_left = -bits_needed & 7; // `-bits_needed mod 8` + + if (n < 64) { + uint64_t mask = (UINT64_C(1) << n) - 1; + res &= mask; + } + // if `n == 64`, do nothing + return res; +} + +// ======================================================================== +// Byte arrays +// ======================================================================== + +std::string kaitai::kstream::read_bytes(std::streamsize len) { + std::vector result(len); + + // NOTE: streamsize type is signed, negative values are only *supposed* to not be used. + // http://en.cppreference.com/w/cpp/io/streamsize + if (len < 0) { + throw std::runtime_error("read_bytes: requested a negative amount"); + } + + if (len > 0) { + m_io->read(&result[0], len); + } + + return std::string(result.begin(), result.end()); +} + +std::string kaitai::kstream::read_bytes_full() { + std::iostream::pos_type p1 = m_io->tellg(); + m_io->seekg(0, std::ios::end); + std::iostream::pos_type p2 = m_io->tellg(); + size_t len = p2 - p1; + + // Note: this requires a std::string to be backed with a + // contiguous buffer. Officially, it's a only requirement since + // C++11 (C++98 and C++03 didn't have this requirement), but all + // major implementations had contiguous buffers anyway. + std::string result(len, ' '); + m_io->seekg(p1); + m_io->read(&result[0], len); + + return result; +} + +std::string kaitai::kstream::read_bytes_term(char term, bool include, bool consume, bool eos_error) { + std::string result; + std::getline(*m_io, result, term); + if (m_io->eof()) { + // encountered EOF + if (eos_error) { + throw std::runtime_error("read_bytes_term: encountered EOF"); + } + } else { + // encountered terminator + if (include) + result.push_back(term); + if (!consume) + m_io->unget(); + } + return result; +} + +std::string kaitai::kstream::ensure_fixed_contents(std::string expected) { + std::string actual = read_bytes(expected.length()); + + if (actual != expected) { + // NOTE: I think printing it outright is not best idea, it could contain non-ASCII characters + // like backspace and beeps and whatnot. It would be better to print hexlified version, and + // also to redirect it to stderr. + throw std::runtime_error("ensure_fixed_contents: actual data does not match expected data"); + } + + return actual; +} + +std::string kaitai::kstream::bytes_strip_right(std::string src, char pad_byte) { + std::size_t new_len = src.length(); + + while (new_len > 0 && src[new_len - 1] == pad_byte) + new_len--; + + return src.substr(0, new_len); +} + +std::string kaitai::kstream::bytes_terminate(std::string src, char term, bool include) { + std::size_t new_len = 0; + std::size_t max_len = src.length(); + + while (new_len < max_len && src[new_len] != term) + new_len++; + + if (include && new_len < max_len) + new_len++; + + return src.substr(0, new_len); +} + +// ======================================================================== +// Byte array processing +// ======================================================================== + +std::string kaitai::kstream::process_xor_one(std::string data, uint8_t key) { + size_t len = data.length(); + std::string result(len, ' '); + + for (size_t i = 0; i < len; i++) + result[i] = data[i] ^ key; + + return result; +} + +std::string kaitai::kstream::process_xor_many(std::string data, std::string key) { + size_t len = data.length(); + size_t kl = key.length(); + std::string result(len, ' '); + + size_t ki = 0; + for (size_t i = 0; i < len; i++) { + result[i] = data[i] ^ key[ki]; + ki++; + if (ki >= kl) + ki = 0; + } + + return result; +} + +std::string kaitai::kstream::process_rotate_left(std::string data, int amount) { + size_t len = data.length(); + std::string result(len, ' '); + + for (size_t i = 0; i < len; i++) { + uint8_t bits = data[i]; + result[i] = (bits << amount) | (bits >> (8 - amount)); + } + + return result; +} + +#ifdef KS_ZLIB +#include + +std::string kaitai::kstream::process_zlib(std::string data) { + int ret; + + unsigned char *src_ptr = reinterpret_cast(&data[0]); + std::stringstream dst_strm; + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + ret = inflateInit(&strm); + if (ret != Z_OK) + throw std::runtime_error("process_zlib: inflateInit error"); + + strm.next_in = src_ptr; + strm.avail_in = data.length(); + + unsigned char outbuffer[ZLIB_BUF_SIZE]; + std::string outstring; + + // get the decompressed bytes blockwise using repeated calls to inflate + do { + strm.next_out = reinterpret_cast(outbuffer); + strm.avail_out = sizeof(outbuffer); + + ret = inflate(&strm, 0); + + if (outstring.size() < strm.total_out) + outstring.append(reinterpret_cast(outbuffer), strm.total_out - outstring.size()); + } while (ret == Z_OK); + + if (ret != Z_STREAM_END) { // an error occurred that was not EOF + std::ostringstream exc_msg; + exc_msg << "process_zlib: error #" << ret << "): " << strm.msg; + throw std::runtime_error(exc_msg.str()); + } + + if (inflateEnd(&strm) != Z_OK) + throw std::runtime_error("process_zlib: inflateEnd error"); + + return outstring; +} +#endif + +// ======================================================================== +// Misc utility methods +// ======================================================================== + +int kaitai::kstream::mod(int a, int b) { + if (b <= 0) + throw std::invalid_argument("mod: divisor b <= 0"); + int r = a % b; + if (r < 0) + r += b; + return r; +} + +#include +void kaitai::kstream::unsigned_to_decimal(uint64_t number, char *buffer) { + // Implementation from https://ideone.com/nrQfA8 by Alf P. Steinbach + // (see https://www.zverovich.net/2013/09/07/integer-to-string-conversion-in-cplusplus.html#comment-1033931478) + if (number == 0) { + *buffer++ = '0'; + } else { + char *p_first = buffer; + while (number != 0) { + *buffer++ = static_cast('0' + number % 10); + number /= 10; + } + std::reverse(p_first, buffer); + } + *buffer = '\0'; +} + +std::string kaitai::kstream::reverse(std::string val) { + std::reverse(val.begin(), val.end()); + + return val; +} + +uint8_t kaitai::kstream::byte_array_min(const std::string val) { + uint8_t min = 0xff; // UINT8_MAX + std::string::const_iterator end = val.end(); + for (std::string::const_iterator it = val.begin(); it != end; ++it) { + uint8_t cur = static_cast(*it); + if (cur < min) { + min = cur; + } + } + return min; +} + +uint8_t kaitai::kstream::byte_array_max(const std::string val) { + uint8_t max = 0; // UINT8_MIN + std::string::const_iterator end = val.end(); + for (std::string::const_iterator it = val.begin(); it != end; ++it) { + uint8_t cur = static_cast(*it); + if (cur > max) { + max = cur; + } + } + return max; +} + +// ======================================================================== +// Other internal methods +// ======================================================================== + +#ifndef KS_STR_DEFAULT_ENCODING +#define KS_STR_DEFAULT_ENCODING "UTF-8" +#endif + +#ifdef KS_STR_ENCODING_ICONV + +#include +#include +#include + +std::string kaitai::kstream::bytes_to_str(std::string src, std::string src_enc) { + iconv_t cd = iconv_open(KS_STR_DEFAULT_ENCODING, src_enc.c_str()); + + if (cd == (iconv_t)-1) { + if (errno == EINVAL) { + throw std::runtime_error("bytes_to_str: invalid encoding pair conversion requested"); + } else { + throw std::runtime_error("bytes_to_str: error opening iconv"); + } + } + + size_t src_len = src.length(); + size_t src_left = src_len; + + // Start with a buffer length of double the source length. + size_t dst_len = src_len * 2; + std::string dst(dst_len, ' '); + size_t dst_left = dst_len; + + char *src_ptr = &src[0]; + char *dst_ptr = &dst[0]; + + while (true) { + size_t res = iconv(cd, &src_ptr, &src_left, &dst_ptr, &dst_left); + + if (res == (size_t)-1) { + if (errno == E2BIG) { + // dst buffer is not enough to accomodate whole string + // enlarge the buffer and try again + size_t dst_used = dst_len - dst_left; + dst_left += dst_len; + dst_len += dst_len; + dst.resize(dst_len); + + // dst.resize might have allocated destination buffer in another area + // of memory, thus our previous pointer "dst" will be invalid; re-point + // it using "dst_used". + dst_ptr = &dst[dst_used]; + } else { + throw std::runtime_error("bytes_to_str: iconv error"); + } + } else { + // conversion successful + dst.resize(dst_len - dst_left); + break; + } + } + + if (iconv_close(cd) != 0) { + throw std::runtime_error("bytes_to_str: iconv close error"); + } + + return dst; +} +#elif defined(KS_STR_ENCODING_NONE) +std::string kaitai::kstream::bytes_to_str(std::string src, std::string src_enc) { + (void)src_enc; + return src; +} +#else +#error Need to decide how to handle strings: please define one of: KS_STR_ENCODING_ICONV, KS_STR_ENCODING_NONE +#endif diff --git a/common/kaitai/kaitaistream.h b/common/kaitai/kaitaistream.h new file mode 100644 index 0000000..3a25a7d --- /dev/null +++ b/common/kaitai/kaitaistream.h @@ -0,0 +1,295 @@ +#ifndef KAITAI_STREAM_H +#define KAITAI_STREAM_H + +// Kaitai Struct runtime API version: x.y.z = 'xxxyyyzzz' decimal +#define KAITAI_STRUCT_VERSION 10000L + +#include +#include +#include +#include +#include + +namespace kaitai { + +/** + * Kaitai Stream class (kaitai::kstream) is an implementation of + * Kaitai Struct stream API + * for C++/STL. It's implemented as a wrapper over generic STL std::istream. + * + * It provides a wide variety of simple methods to read (parse) binary + * representations of primitive types, such as integer and floating + * point numbers, byte arrays and strings, and also provides stream + * positioning / navigation methods with unified cross-language and + * cross-toolkit semantics. + * + * Typically, end users won't access Kaitai Stream class manually, but would + * describe a binary structure format using .ksy language and then would use + * Kaitai Struct compiler to generate source code in desired target language. + * That code, in turn, would use this class and API to do the actual parsing + * job. + */ +class kstream { +public: + /** + * Constructs new Kaitai Stream object, wrapping a given std::istream. + * \param io istream object to use for this Kaitai Stream + */ + kstream(std::istream* io); + + /** + * Constructs new Kaitai Stream object, wrapping a given in-memory data + * buffer. + * \param data data buffer to use for this Kaitai Stream + */ + kstream(const std::string& data); + + void close(); + + /** @name Stream positioning */ + //@{ + /** + * Check if stream pointer is at the end of stream. Note that the semantics + * are different from traditional STL semantics: one does *not* need to do a + * read (which will fail) after the actual end of the stream to trigger EOF + * flag, which can be accessed after that read. It is sufficient to just be + * at the end of the stream for this method to return true. + * \return "true" if we are located at the end of the stream. + */ + bool is_eof() const; + + /** + * Set stream pointer to designated position. + * \param pos new position (offset in bytes from the beginning of the stream) + */ + void seek(uint64_t pos); + + /** + * Get current position of a stream pointer. + * \return pointer position, number of bytes from the beginning of the stream + */ + uint64_t pos(); + + /** + * Get total size of the stream in bytes. + * \return size of the stream in bytes + */ + uint64_t size(); + //@} + + /** @name Integer numbers */ + //@{ + + // ------------------------------------------------------------------------ + // Signed + // ------------------------------------------------------------------------ + + int8_t read_s1(); + + // ........................................................................ + // Big-endian + // ........................................................................ + + int16_t read_s2be(); + int32_t read_s4be(); + int64_t read_s8be(); + + // ........................................................................ + // Little-endian + // ........................................................................ + + int16_t read_s2le(); + int32_t read_s4le(); + int64_t read_s8le(); + + // ------------------------------------------------------------------------ + // Unsigned + // ------------------------------------------------------------------------ + + uint8_t read_u1(); + + // ........................................................................ + // Big-endian + // ........................................................................ + + uint16_t read_u2be(); + uint32_t read_u4be(); + uint64_t read_u8be(); + + // ........................................................................ + // Little-endian + // ........................................................................ + + uint16_t read_u2le(); + uint32_t read_u4le(); + uint64_t read_u8le(); + + //@} + + /** @name Floating point numbers */ + //@{ + + // ........................................................................ + // Big-endian + // ........................................................................ + + float read_f4be(); + double read_f8be(); + + // ........................................................................ + // Little-endian + // ........................................................................ + + float read_f4le(); + double read_f8le(); + + //@} + + /** @name Unaligned bit values */ + //@{ + + void align_to_byte(); + uint64_t read_bits_int_be(int n); + uint64_t read_bits_int(int n); + uint64_t read_bits_int_le(int n); + + //@} + + /** @name Byte arrays */ + //@{ + + std::string read_bytes(std::streamsize len); + std::string read_bytes_full(); + std::string read_bytes_term(char term, bool include, bool consume, bool eos_error); + std::string ensure_fixed_contents(std::string expected); + + static std::string bytes_strip_right(std::string src, char pad_byte); + static std::string bytes_terminate(std::string src, char term, bool include); + static std::string bytes_to_str(std::string src, std::string src_enc); + + //@} + + /** @name Byte array processing */ + //@{ + + /** + * Performs a XOR processing with given data, XORing every byte of input with a single + * given value. + * @param data data to process + * @param key value to XOR with + * @return processed data + */ + static std::string process_xor_one(std::string data, uint8_t key); + + /** + * Performs a XOR processing with given data, XORing every byte of input with a key + * array, repeating key array many times, if necessary (i.e. if data array is longer + * than key array). + * @param data data to process + * @param key array of bytes to XOR with + * @return processed data + */ + static std::string process_xor_many(std::string data, std::string key); + + /** + * Performs a circular left rotation shift for a given buffer by a given amount of bits, + * using groups of 1 bytes each time. Right circular rotation should be performed + * using this procedure with corrected amount. + * @param data source data to process + * @param amount number of bits to shift by + * @return copy of source array with requested shift applied + */ + static std::string process_rotate_left(std::string data, int amount); + +#ifdef KS_ZLIB + /** + * Performs an unpacking ("inflation") of zlib-compressed data with usual zlib headers. + * @param data data to unpack + * @return unpacked data + * @throws IOException + */ + static std::string process_zlib(std::string data); +#endif + + //@} + + /** + * Performs modulo operation between two integers: dividend `a` + * and divisor `b`. Divisor `b` is expected to be positive. The + * result is always 0 <= x <= b - 1. + */ + static int mod(int a, int b); + + /** + * Converts given integer `val` to a decimal string representation. + * Should be used in place of std::to_string() (which is available only + * since C++11) in older C++ implementations. + */ + template +// check for C++11 support - https://stackoverflow.com/a/40512515 +#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1900) + // https://stackoverflow.com/a/27913885 + typename std::enable_if< + std::is_integral::value && + // check if we don't have something too large like GCC's `__int128_t` + std::numeric_limits::max() >= 0 && + std::numeric_limits::max() <= std::numeric_limits::max(), + std::string + >::type +#else + std::string +#endif + static to_string(I val) { + // in theory, `digits10 + 3` would be enough (minus sign + leading digit + // + null terminator), but let's add a little more to be safe + char buf[std::numeric_limits::digits10 + 5]; + if (val < 0) { + buf[0] = '-'; + // get absolute value without undefined behavior (https://stackoverflow.com/a/12231604) + unsigned_to_decimal(-static_cast(val), &buf[1]); + } else { + unsigned_to_decimal(val, buf); + } + return std::string(buf); + } + + /** + * Reverses given string `val`, so that the first character becomes the + * last and the last one becomes the first. This should be used to avoid + * the need of local variables at the caller. + */ + static std::string reverse(std::string val); + + /** + * Finds the minimal byte in a byte array, treating bytes as + * unsigned values. + * @param val byte array to scan + * @return minimal byte in byte array as integer + */ + static uint8_t byte_array_min(const std::string val); + + /** + * Finds the maximal byte in a byte array, treating bytes as + * unsigned values. + * @param val byte array to scan + * @return maximal byte in byte array as integer + */ + static uint8_t byte_array_max(const std::string val); + +private: + std::istream* m_io; + std::istringstream m_io_str; + int m_bits_left; + uint64_t m_bits; + + void init(); + void exceptions_enable() const; + + static void unsigned_to_decimal(uint64_t number, char *buffer); + + static const int ZLIB_BUF_SIZE = 128 * 1024; +}; + +} + +#endif diff --git a/common/kaitai/kaitaistruct.h b/common/kaitai/kaitaistruct.h new file mode 100644 index 0000000..7b5acd6 --- /dev/null +++ b/common/kaitai/kaitaistruct.h @@ -0,0 +1,20 @@ +#ifndef KAITAI_STRUCT_H +#define KAITAI_STRUCT_H + +#include "kaitaistream.h" + +namespace kaitai { + +class kstruct { +public: + kstruct(kstream *_io) { m__io = _io; } + virtual ~kstruct() {} +protected: + kstream *m__io; +public: + kstream *_io() { return m__io; } +}; + +} + +#endif diff --git a/common/ksy/intel_acbp_v1.ksy b/common/ksy/intel_acbp_v1.ksy new file mode 100644 index 0000000..326a3da --- /dev/null +++ b/common/ksy/intel_acbp_v1.ksy @@ -0,0 +1,202 @@ +meta: + id: intel_acbp_v1 + title: Intel BootGuard Boot Policy v1 + application: Intel x86 firmware + file-extension: acbp_v1 + tags: + - firmware + license: CC0-1.0 + ks-version: 0.9 + endian: le + +enums: + ibb_segment_type: + 0: ibb + 1: non_ibb + + structure_ids: + 0x5f5f504243415f5f: acbp + 0x5f5f534242495f5f: ibbs + 0x5f5f41444d505f5f: pmda + 0x5f5f47534d505f5f: pmsg + +seq: +- id: structure_id + type: u8 + enum: structure_ids + valid: structure_ids::acbp +- id: version + type: u1 + valid: + expr: _ < 0x20 +- id: reserved0 + type: u1 +- id: bpm_revision + type: u1 +- id: bp_svn + type: u1 +- id: acm_svn + type: u1 +- id: reserved1 + type: u1 +- id: nem_data_size + type: u2 +- id: elements + type: acbp_element + repeat: until + repeat-until: _.header.structure_id == structure_ids::pmsg or _io.eof + +types: + acbp_element: + seq: + - id: header + type: common_header + - id: ibbs_body + type: ibbs_body + if: header.structure_id == structure_ids::ibbs + - id: pmda_body + type: pmda_body + if: header.structure_id == structure_ids::pmda + - id: pmsg_body + type: pmsg_body + if: header.structure_id == structure_ids::pmsg + - id: invalid_body + size: 0 + if: header.structure_id != structure_ids::pmsg + and header.structure_id != structure_ids::pmda + and header.structure_id != structure_ids::ibbs + valid: + expr: false + + common_header: + seq: + - id: structure_id + type: u8 + enum: structure_ids + - id: version + type: u1 + + hash: + seq: + - id: hash_algorithm_id + type: u2 + - id: len_hash + type: u2 + - id: hash + size: 32 + + ibbs_body: + seq: + - id: reserved + type: u1 + repeat: expr + repeat-expr: 3 + - id: flags + type: u4 + - id: mch_bar + type: u8 + - id: vtd_bar + type: u8 + - id: dma_protection_base0 + type: u4 + - id: dma_protection_limit0 + type: u4 + - id: dma_protection_base1 + type: u8 + - id: dma_protection_limit1 + type: u8 + - id: post_ibb_hash + type: hash + - id: ibb_entry_point + type: u4 + - id: ibb_hash + type: hash + - id: num_ibb_segments + type: u1 + - id: ibb_segments + type: ibb_segment + repeat: expr + repeat-expr: num_ibb_segments + + ibb_segment: + seq: + - id: reserved + type: u2 + - id: flags + type: u2 + - id: base + type: u4 + - id: size + type: u4 + + pmda_body: + seq: + - id: total_size + type: u2 + - id: version + type: u4 + - id: num_entries + type: u4 + - id: entries_v1 + if: version == 1 + type: pmda_entry_v1 + repeat: expr + repeat-expr: num_entries + - id: entries_v2 + if: version == 2 + type: pmda_entry_v2 + repeat: expr + repeat-expr: num_entries + + pmda_entry_v1: + seq: + - id: base + type: u4 + - id: size + type: u4 + - id: hash + size: 32 + + pmda_entry_v2: + seq: + - id: base + type: u4 + - id: size + type: u4 + - id: hash + type: hash + + pmsg_body: + seq: + - id: version + type: u1 + - id: key_id + type: u2 + - id: public_key + type: public_key + - id: sig_scheme + type: u2 + - id: signature + type: signature + + public_key: + seq: + - id: version + type: u1 + - id: size_bits + type: u2 + - id: exponent + type: u4 + - id: modulus + size: size_bits / 8 + + signature: + seq: + - id: version + type: u1 + - id: size_bits + type: u2 + - id: hash_algorithm_id + type: u2 + - id: signature + size: size_bits / 8 \ No newline at end of file diff --git a/common/ksy/intel_acbp_v2.ksy b/common/ksy/intel_acbp_v2.ksy new file mode 100644 index 0000000..2782bdd --- /dev/null +++ b/common/ksy/intel_acbp_v2.ksy @@ -0,0 +1,223 @@ +meta: + id: intel_acbp_v2 + title: Intel BootGuard Boot Policy v2 + application: Intel x86 firmware + file-extension: acbp_v2 + tags: + - firmware + license: CC0-1.0 + ks-version: 0.9 + endian: le + +enums: + ibb_segment_type: + 0: ibb + 1: non_ibb + + structure_ids: + 0x5f5f504243415f5f: acbp + 0x5f5f534242495f5f: ibbs + 0x5f5f535458545f5f: txts + 0x5f5f535246505f5f: pfrs + 0x5f5f534443505f5f: pcds + 0x5f5f41444d505f5f: pmda + 0x5f5f47534d505f5f: pmsg + +seq: +- id: structure_id + type: u8 + enum: structure_ids + valid: structure_ids::acbp +- id: version + type: u1 + valid: + expr: _ >= 0x20 +- id: header_specific + type: u1 +- id: total_size + type: u2 + valid: 0x14 +- id: key_signature_offset + type: u2 +- id: bpm_revision + type: u1 +- id: bp_svn + type: u1 +- id: acm_svn + type: u1 +- id: reserved + type: u1 +- id: nem_data_size + type: u2 +- id: elements + type: acbp_element + repeat: until + repeat-until: _.header.total_size == 0 or _.header.structure_id == structure_ids::pmsg +- id: key_signature + type: key_signature + +types: + header: + seq: + - id: structure_id + type: u8 + enum: structure_ids + - id: version + type: u1 + - id: header_specific + type: u1 + - id: total_size + type: u2 + + hash: + seq: + - id: hash_algorithm_id + type: u2 + - id: len_hash + type: u2 + - id: hash + size: len_hash + + pmda_entry_v3: + seq: + - id: entry_id + type: u4 + - id: base + type: u4 + - id: size + type: u4 + - id: total_entry_size + type: u2 + - id: version + type: u2 + valid: 3 + - id: hash + type: hash + + pmda_body: + seq: + - id: reserved + type: u2 + - id: total_size + type: u2 + - id: version + type: u4 + - id: num_entries + type: u4 + - id: entries + type: pmda_entry_v3 + repeat: expr + repeat-expr: num_entries + + ibb_segment: + seq: + - id: reserved + type: u2 + - id: flags + type: u2 + - id: base + type: u4 + - id: size + type: u4 + + ibbs_body: + seq: + - id: reserved0 + type: u1 + - id: set_number + type: u1 + - id: reserved1 + type: u1 + - id: pbet_value + type: u1 + - id: flags + type: u4 + - id: mch_bar + type: u8 + - id: vtd_bar + type: u8 + - id: dma_protection_base0 + type: u4 + - id: dma_protection_limit0 + type: u4 + - id: dma_protection_base1 + type: u8 + - id: dma_protection_limit1 + type: u8 + - id: post_ibb_digest + type: hash + - id: ibb_entry_point + type: u4 + - id: ibb_digests_size + type: u2 + - id: num_ibb_digests + type: u2 + - id: ibb_digests + type: hash + repeat: expr + repeat-expr: num_ibb_digests + - id: obb_digest + type: hash + - id: reserved2 + type: u1 + repeat: expr + repeat-expr: 3 + - id: num_ibb_segments + type: u1 + - id: ibb_segments + type: ibb_segment + repeat: expr + repeat-expr: num_ibb_segments + + acbp_element: + seq: + - id: header + type: header + - id: ibbs_body + type: ibbs_body + if: header.structure_id == structure_ids::ibbs + and header.total_size >= sizeof
+ - id: pmda_body + type: pmda_body + if: header.structure_id == structure_ids::pmda + and header.total_size >= sizeof
+ - id: generic_body + size: header.total_size - sizeof
+ if: header.structure_id != structure_ids::ibbs + and header.structure_id != structure_ids::pmda + and header.total_size >= sizeof
+ + public_key: + seq: + - id: version + type: u1 + - id: size_bits + type: u2 + - id: exponent + type: u4 + - id: modulus + size: size_bits / 8 + + signature: + seq: + - id: version + type: u1 + - id: size_bits + type: u2 + - id: hash_algorithm_id + type: u2 + - id: signature + size: size_bits / 8 + + key_signature: + seq: + - id: version + type: u1 + - id: key_id + type: u2 + - id: public_key + type: public_key + - id: sig_scheme + type: u2 + - id: signature + type: signature \ No newline at end of file diff --git a/common/ksy/intel_acm.ksy b/common/ksy/intel_acm.ksy new file mode 100644 index 0000000..a4d8c4f --- /dev/null +++ b/common/ksy/intel_acm.ksy @@ -0,0 +1,94 @@ +meta: + id: intel_acm + title: Intel Authenticated Code Module + application: Intel x86 firmware + file-extension: acm + tags: + - executable + - firmware + license: CC0-1.0 + ks-version: 0.9 + endian: le + +enums: + module_subtype: + 0: txt + 1: startup + 3: boot_guard + + known_header_version: + 0x00000000: v0_0 + 0x00030000: v3_0 + +seq: +- id: header + type: header +- id: body + size: 4 * (header.module_size - header.header_size - header.scratch_space_size) + +types: + header: + seq: + - id: module_type + type: u2 + valid: 0x0002 + - id: module_subtype + type: u2 + enum: module_subtype + - id: header_size + type: u4 + doc: counted in 4 byte increments + - id: header_version + type: u4 + - id: chipset_id + type: u2 + - id: flags + type: u2 + - id: module_vendor + type: u4 + valid: 0x8086 + - id: date_day + type: u1 + doc: BCD + - id: date_month + type: u1 + doc: BCD + - id: date_year + type: u2 + doc: BCD + - id: module_size + type: u4 + doc: counted in 4 byte increments + - id: acm_svn + type: u2 + - id: se_svn + type: u2 + - id: code_control_flags + type: u4 + - id: error_entry_point + type: u4 + - id: gdt_max + type: u4 + - id: gdt_base + type: u4 + - id: segment_sel + type: u4 + - id: entry_point + type: u4 + - id: reserved + size: 64 + - id: key_size + type: u4 + doc: counted in 4 byte increments + - id: scratch_space_size + type: u4 + doc: counted in 4 byte increments + - id: rsa_public_key + size: (4 * key_size) + - id: rsa_exponent + type: u4 + if: header_version == 0 + - id: rsa_signature + size: (4 * key_size) + - id: scratch_space + size: (4 * scratch_space_size) diff --git a/common/ksy/intel_keym_v1.ksy b/common/ksy/intel_keym_v1.ksy new file mode 100644 index 0000000..431856d --- /dev/null +++ b/common/ksy/intel_keym_v1.ksy @@ -0,0 +1,79 @@ +meta: + id: intel_keym_v1 + title: Intel BootGuard Key Manifest v1 + application: Intel x86 firmware + file-extension: keym_v1 + tags: + - firmware + license: CC0-1.0 + ks-version: 0.9 + endian: le + +enums: + structure_ids: + 0x5f5f4d59454b5f5f: keym + +seq: +- id: structure_id + type: u8 + enum: structure_ids + valid: structure_ids::keym +- id: version + type: u1 + valid: + expr: _ < 0x20 +- id: km_version + type: u1 +- id: km_svn + type: u1 +- id: km_id + type: u1 +- id: km_hash + type: km_hash +- id: key_signature + type: key_signature + +types: + km_hash: + seq: + - id: hash_algorithm_id + type: u2 + - id: len_hash + type: u2 + - id: hash + size: len_hash + + public_key: + seq: + - id: version + type: u1 + - id: size_bits + type: u2 + - id: exponent + type: u4 + - id: modulus + size: size_bits / 8 + + signature: + seq: + - id: version + type: u1 + - id: size_bits + type: u2 + - id: hash_algorithm_id + type: u2 + - id: signature + size: size_bits / 8 + + key_signature: + seq: + - id: version + type: u1 + - id: key_id + type: u2 + - id: public_key + type: public_key + - id: sig_scheme + type: u2 + - id: signature + type: signature \ No newline at end of file diff --git a/common/ksy/intel_keym_v2.ksy b/common/ksy/intel_keym_v2.ksy new file mode 100644 index 0000000..0fd3d87 --- /dev/null +++ b/common/ksy/intel_keym_v2.ksy @@ -0,0 +1,109 @@ +meta: + id: intel_keym_v2 + title: Intel BootGuard Key Manifest v2 + application: Intel x86 firmware + file-extension: keym_v2 + tags: + - firmware + license: CC0-1.0 + ks-version: 0.9 + endian: le + +enums: + structure_ids: + 0x5f5f4d59454b5f5f: keym + + km_usage_flags: + 1: boot_policy_manifest + 2: fit_patch_manifest + 4: acm_manifest + 8: sdev + +seq: +- id: header + type: header +- id: key_signature_offset + type: u2 +- id: reserved + type: u1 + repeat: expr + repeat-expr: 3 +- id: km_version + type: u1 +- id: km_svn + type: u1 +- id: km_id + type: u1 +- id: fpf_hash_algorithm_id + type: u2 +- id: num_km_hashes + type: u2 +- id: km_hashes + type: km_hash + repeat: expr + repeat-expr: num_km_hashes +- id: key_signature + type: key_signature + +types: + header: + seq: + - id: structure_id + type: u8 + enum: structure_ids + valid: structure_ids::keym + - id: version + type: u1 + valid: + expr: _ >= 0x20 + - id: header_specific + type: u1 + - id: total_size + type: u2 + valid: 0x0 + + km_hash: + seq: + - id: usage_flags + type: u8 + - id: hash_algorithm_id + type: u2 + - id: len_hash + type: u2 + - id: hash + size: len_hash + + public_key: + seq: + - id: version + type: u1 + - id: size_bits + type: u2 + - id: exponent + type: u4 + - id: modulus + size: size_bits / 8 + + signature: + seq: + - id: version + type: u1 + - id: size_bits + type: u2 + - id: hash_algorithm_id + type: u2 + - id: signature + size: size_bits / 8 + + key_signature: + seq: + - id: version + type: u1 + - id: key_id + type: u2 + - id: public_key + type: public_key + - id: sig_scheme + type: u2 + - id: signature + type: signature \ No newline at end of file diff --git a/common/me.h b/common/me.h index 74f4b7b..6507815 100644 --- a/common/me.h +++ b/common/me.h @@ -27,12 +27,12 @@ typedef struct ME_VERSION_ { UINT16 Build; } ME_VERSION; -const UByteArray ME_VERSION_SIGNATURE("\x24\x4D\x41\x4E", 4); //$MAN -const UByteArray ME_VERSION_SIGNATURE2("\x24\x4D\x4E\x32", 4); //$MN2 +#define ME_VERSION_SIGNATURE 0x4E414D24 //$MAN +#define ME_VERSION_SIGNATURE2 0x324E4D24 //$MN2 // FPT #define ME_ROM_BYPASS_VECTOR_SIZE 0x10 -const UByteArray FPT_HEADER_SIGNATURE("\x24\x46\x50\x54", 4); //$FPT +#define FPT_HEADER_SIGNATURE 0x54504624 //$FPT // Header version 1.0 or 2.0, default typedef struct FPT_HEADER_ { diff --git a/common/meparser.cpp b/common/meparser.cpp index 08733f9..8db1433 100755 --- a/common/meparser.cpp +++ b/common/meparser.cpp @@ -19,7 +19,6 @@ #include "meparser.h" #include "parsingdata.h" #include "utility.h" -#include "uinttypes.h" #ifdef U_ENABLE_ME_PARSING_SUPPORT @@ -54,7 +53,7 @@ USTATUS MeParser::parseMeRegionBody(const UModelIndex & index) // Check ME signature to determine it's version // ME v11 and older layout - if (meRegion.left(sizeof(UINT32)) == FPT_HEADER_SIGNATURE || meRegion.mid(ME_ROM_BYPASS_VECTOR_SIZE, sizeof(UINT32)) == FPT_HEADER_SIGNATURE) { + if (*(UINT32*)meRegion.constData() == FPT_HEADER_SIGNATURE || *(UINT32*)(meRegion.constData() + ME_ROM_BYPASS_VECTOR_SIZE) == FPT_HEADER_SIGNATURE) { UModelIndex ptIndex; return parseFptRegion(meRegion, index, ptIndex); } @@ -73,7 +72,7 @@ USTATUS MeParser::parseMeRegionBody(const UModelIndex & index) return U_INVALID_ME_PARTITION_TABLE; } // Data partition always points to FPT header - if (meRegion.mid(ifwi16Header->DataPartition.Offset, sizeof(UINT32)) == FPT_HEADER_SIGNATURE) { + if (*(UINT32*)(meRegion.constData() + ifwi16Header->DataPartition.Offset) == FPT_HEADER_SIGNATURE) { UModelIndex ptIndex; return parseIfwi16Region(meRegion, index, ptIndex); } @@ -91,7 +90,7 @@ USTATUS MeParser::parseMeRegionBody(const UModelIndex & index) return U_INVALID_ME_PARTITION_TABLE; } // Data partition always points to FPT header - if (meRegion.mid(ifwi17Header->DataPartition.Offset, sizeof(UINT32)) == FPT_HEADER_SIGNATURE) { + if (*(UINT32*)(meRegion.constData() + ifwi17Header->DataPartition.Offset)== FPT_HEADER_SIGNATURE) { UModelIndex ptIndex; return parseIfwi17Region(meRegion, index, ptIndex); } @@ -112,7 +111,7 @@ USTATUS MeParser::parseFptRegion(const UByteArray & region, const UModelIndex & // Populate partition table header const FPT_HEADER* ptHeader = (const FPT_HEADER*)region.constData(); UINT32 romBypassVectorSize = 0; - if (region.left(sizeof(UINT32)) != FPT_HEADER_SIGNATURE) { + if (*(UINT32*)region.constData() != FPT_HEADER_SIGNATURE) { // Adjust the header to skip ROM bypass vector romBypassVectorSize = ME_ROM_BYPASS_VECTOR_SIZE; ptHeader = (const FPT_HEADER*)(region.constData() + romBypassVectorSize); diff --git a/common/meparser.h b/common/meparser.h index c9fff4b..db26505 100755 --- a/common/meparser.h +++ b/common/meparser.h @@ -22,7 +22,8 @@ WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. #include "ubytearray.h" #include "treemodel.h" #include "ffsparser.h" -#include "sha256.h" + +#include "digest/sha2.h" #ifdef U_ENABLE_ME_PARSING_SUPPORT class MeParser diff --git a/common/meson.build b/common/meson.build index ec125d5..0521c50 100644 --- a/common/meson.build +++ b/common/meson.build @@ -23,6 +23,7 @@ uefitoolcommon = static_library('uefitoolcommon', 'nvram.cpp', 'nvramparser.cpp', 'meparser.cpp', + 'fitparser.cpp', 'ffsparser.cpp', 'ffsreport.cpp', 'ffsutils.cpp', @@ -31,7 +32,16 @@ uefitoolcommon = static_library('uefitoolcommon', 'treemodel.cpp', 'utility.cpp', 'ustring.cpp', - 'sha256.c', + 'generated/intel_acbp_v1.cpp', + 'generated/intel_acbp_v2.cpp', + 'generated/intel_keym_v1.cpp', + 'generated/intel_keym_v2.cpp', + 'generated/intel_acm.cpp', + 'kaitai/kaitaistream.cpp', + 'digest/sha1.c', + 'digest/sha256.c', + 'digest/sha512.c', + 'digest/sm3.c', ], cpp_args: [ '-DU_ENABLE_NVRAM_PARSING_SUPPORT', diff --git a/common/nvram.cpp b/common/nvram.cpp index a914153..93bf7a9 100644 --- a/common/nvram.cpp +++ b/common/nvram.cpp @@ -12,6 +12,66 @@ */ #include "nvram.h" +#include "ubytearray.h" + +// +// GUIDs mentioned in by nvram.h +// +extern const UByteArray NVRAM_NVAR_STORE_FILE_GUID // CEF5B9A3-476D-497F-9FDC-E98143E0422C +("\xA3\xB9\xF5\xCE\x6D\x47\x7F\x49\x9F\xDC\xE9\x81\x43\xE0\x42\x2C", 16); +extern const UByteArray NVRAM_NVAR_EXTERNAL_DEFAULTS_FILE_GUID // 9221315B-30BB-46B5-813E-1B1BF4712BD3 +("\x5B\x31\x21\x92\xBB\x30\xB5\x46\x81\x3E\x1B\x1B\xF4\x71\x2B\xD3", 16); +extern const UByteArray NVRAM_NVAR_PEI_EXTERNAL_DEFAULTS_FILE_GUID // 77D3DC50-D42B-4916-AC80-8F469035D150 +("\x50\xDC\xD3\x77\x2B\xD4\x16\x49\xAC\x80\x8F\x46\x90\x35\xD1\x50", 16); +extern const UByteArray NVRAM_NVAR_BB_DEFAULTS_FILE_GUID // AF516361-B4C5-436E-A7E3-A149A31B1461 +("\x61\x63\x51\xAF\xC5\xB4\x6E\x43\xA7\xE3\xA1\x49\xA3\x1B\x14\x61", 16); +extern const UByteArray NVRAM_MAIN_STORE_VOLUME_GUID // FFF12B8D-7696-4C8B-A985-2747075B4F50 +("\x8D\x2B\xF1\xFF\x96\x76\x8B\x4C\xA9\x85\x27\x47\x07\x5B\x4F\x50", 16); +extern const UByteArray NVRAM_ADDITIONAL_STORE_VOLUME_GUID // 00504624-8A59-4EEB-BD0F-6B36E96128E0 +("\x24\x46\x50\x00\x59\x8A\xEB\x4E\xBD\x0F\x6B\x36\xE9\x61\x28\xE0", 16); +extern const UByteArray NVRAM_VSS2_AUTH_VAR_KEY_DATABASE_GUID // AAF32C78-947B-439A-A180-2E144EC37792 +("\x78\x2C\xF3\xAA\x7B\x94\x9A\x43\xA1\x80\x2E\x14\x4E\xC3\x77\x92"); +extern const UByteArray NVRAM_VSS2_STORE_GUID // DDCF3617-3275-4164-98B6-FE85707FFE7D +("\x17\x36\xCF\xDD\x75\x32\x64\x41\x98\xB6\xFE\x85\x70\x7F\xFE\x7D"); +extern const UByteArray NVRAM_FDC_STORE_GUID // DDCF3616-3275-4164-98B6-FE85707FFE7D +("\x16\x36\xCF\xDD\x75\x32\x64\x41\x98\xB6\xFE\x85\x70\x7F\xFE\x7D"); +extern const UByteArray EDKII_WORKING_BLOCK_SIGNATURE_GUID // 9E58292B-7C68-497D-0ACE6500FD9F1B95 +("\x2B\x29\x58\x9E\x68\x7C\x7D\x49\x0A\xCE\x65\x00\xFD\x9F\x1B\x95", 16); +extern const UByteArray VSS2_WORKING_BLOCK_SIGNATURE_GUID // 9E58292B-7C68-497D-A0CE6500FD9F1B95 +("\x2B\x29\x58\x9E\x68\x7C\x7D\x49\xA0\xCE\x65\x00\xFD\x9F\x1B\x95", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_VOLUME_HEADER // B091E7D2-05A0-4198-94F0-74B7B8C55459 +("\xD2\xE7\x91\xB0\xA0\x05\x98\x41\x94\xF0\x74\xB7\xB8\xC5\x54\x59", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_MICROCODES_GUID // FD3F690E-B4B0-4D68-89DB-19A1A3318F90 +("\x0E\x69\x3F\xFD\xB0\xB4\x68\x4D\x89\xDB\x19\xA1\xA3\x31\x8F\x90", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_CMDB_GUID // 46310243-7B03-4132-BE44-2243FACA7CDD +("\x43\x02\x31\x46\x03\x7B\x32\x41\xBE\x44\x22\x43\xFA\xCA\x7C\xDD", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_PUBKEY1_GUID // 1B2C4952-D778-4B64-BDA1-15A36F5FA545 +("\x52\x49\x2C\x1B\x78\xD7\x64\x4B\xBD\xA1\x15\xA3\x6F\x5F\xA5\x45", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_MARKER1_GUID // 127C1C4E-9135-46E3-B006-F9808B0559A5 +("\x4E\x1C\x7C\x12\x35\x91\xE3\x46\xB0\x06\xF9\x80\x8B\x05\x59\xA5", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_PUBKEY2_GUID // 7CE75114-8272-45AF-B536-761BD38852CE +("\x14\x51\xE7\x7C\x72\x82\xAF\x45\xB5\x36\x76\x1B\xD3\x88\x52\xCE", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_MARKER2_GUID // 071A3DBE-CFF4-4B73-83F0-598C13DCFDD5 +("\xBE\x3D\x1A\x07\xF4\xCF\x73\x4B\x83\xF0\x59\x8C\x13\xDC\xFD\xD5", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA1_GUID // FACFB110-7BFD-4EFB-873E-88B6B23B97EA +("\x10\xB1\xCF\xFA\xFD\x7B\xFB\x4E\x87\x3E\x88\xB6\xB2\x3B\x97\xEA", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA2_GUID // E68DC11A-A5F4-4AC3-AA2E-29E298BFF645 +("\x1A\xC1\x8D\xE6\xF4\xA5\xC3\x4A\xAA\x2E\x29\xE2\x98\xBF\xF6\x45", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA3_GUID // 4B3828AE-0ACE-45B6-8CDB-DAFC28BBF8C5 +("\xAE\x28\x38\x4B\xCE\x0A\xB6\x45\x8C\xDB\xDA\xFC\x28\xBB\xF8\xC5", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA4_GUID // C22E6B8A-8159-49A3-B353-E84B79DF19C0 +("\x8A\x6B\x2E\xC2\x59\x81\xA3\x49\xB3\x53\xE8\x4B\x79\xDF\x19\xC0", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA5_GUID // B6B5FAB9-75C4-4AAE-8314-7FFFA7156EAA +("\xB9\xFA\xB5\xB6\xC4\x75\xAE\x4A\x83\x14\x7F\xFF\xA7\x15\x6E\xAA", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA6_GUID // 919B9699-8DD0-4376-AA0B-0E54CCA47D8F +("\x99\x96\x9B\x91\xD0\x8D\x76\x43\xAA\x0B\x0E\x54\xCC\xA4\x7D\x8F", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA7_GUID // 58A90A52-929F-44F8-AC35-A7E1AB18AC91 +("\x52\x0A\xA9\x58\x9F\x92\xF8\x44\xAC\x35\xA7\xE1\xAB\x18\xAC\x91", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_SELF_GUID // 8CB71915-531F-4AF5-82BF-A09140817BAA +("\x15\x19\xB7\x8C\x1F\x53\xF5\x4A\x82\xBF\xA0\x91\x40\x81\x7B\xAA", 16); + +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_SIGNATURE +("\x5F\x46\x4C\x41\x53\x48\x5F\x4D\x41\x50", 10); UString nvarAttributesToUString(const UINT8 attributes) { diff --git a/common/nvram.h b/common/nvram.h index d7f8d4e..0ab72b1 100755 --- a/common/nvram.h +++ b/common/nvram.h @@ -25,22 +25,10 @@ WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. // // NVAR store and entry // - -// CEF5B9A3-476D-497F-9FDC-E98143E0422C -const UByteArray NVRAM_NVAR_STORE_FILE_GUID -("\xA3\xB9\xF5\xCE\x6D\x47\x7F\x49\x9F\xDC\xE9\x81\x43\xE0\x42\x2C", 16); - -// 9221315B-30BB-46B5-813E-1B1BF4712BD3 -const UByteArray NVRAM_NVAR_EXTERNAL_DEFAULTS_FILE_GUID -("\x5B\x31\x21\x92\xBB\x30\xB5\x46\x81\x3E\x1B\x1B\xF4\x71\x2B\xD3", 16); - -// 77D3DC50-D42B-4916-AC80-8F469035D150 -const UByteArray NVRAM_NVAR_PEI_EXTERNAL_DEFAULTS_FILE_GUID -("\x50\xDC\xD3\x77\x2B\xD4\x16\x49\xAC\x80\x8F\x46\x90\x35\xD1\x50", 16); - -// AF516361-B4C5-436E-A7E3-A149A31B1461 -const UByteArray NVRAM_NVAR_BB_DEFAULTS_FILE_GUID -("\x61\x63\x51\xAF\xC5\xB4\x6E\x43\xA7\xE3\xA1\x49\xA3\x1B\x14\x61", 16); +extern const UByteArray NVRAM_NVAR_STORE_FILE_GUID; // CEF5B9A3-476D-497F-9FDC-E98143E0422C +extern const UByteArray NVRAM_NVAR_EXTERNAL_DEFAULTS_FILE_GUID; // 9221315B-30BB-46B5-813E-1B1BF4712BD3 +extern const UByteArray NVRAM_NVAR_PEI_EXTERNAL_DEFAULTS_FILE_GUID; // 77D3DC50-D42B-4916-AC80-8F469035D150 +extern const UByteArray NVRAM_NVAR_BB_DEFAULTS_FILE_GUID; // AF516361-B4C5-436E-A7E3-A149A31B1461 extern UString nvarAttributesToUString(const UINT8 attributes); extern UString nvarExtendedAttributesToUString(const UINT8 attributes); @@ -75,14 +63,8 @@ typedef struct NVAR_ENTRY_HEADER_ { // // TianoCore VSS store and variables // - -// FFF12B8D-7696-4C8B-A985-2747075B4F50 -const UByteArray NVRAM_MAIN_STORE_VOLUME_GUID -("\x8D\x2B\xF1\xFF\x96\x76\x8B\x4C\xA9\x85\x27\x47\x07\x5B\x4F\x50", 16); - -// 00504624-8A59-4EEB-BD0F-6B36E96128E0 -const UByteArray NVRAM_ADDITIONAL_STORE_VOLUME_GUID -("\x24\x46\x50\x00\x59\x8A\xEB\x4E\xBD\x0F\x6B\x36\xE9\x61\x28\xE0", 16); +extern const UByteArray NVRAM_MAIN_STORE_VOLUME_GUID; // FFF12B8D-7696-4C8B-A985-2747075B4F50 +extern const UByteArray NVRAM_ADDITIONAL_STORE_VOLUME_GUID; // 00504624-8A59-4EEB-BD0F-6B36E96128E0 #define NVRAM_VSS_STORE_SIGNATURE 0x53535624 // $VSS #define NVRAM_APPLE_SVS_STORE_SIGNATURE 0x53565324 // $SVS @@ -182,18 +164,13 @@ extern UString vssAttributesToUString(const UINT32 attributes); // // VSS2 variables // - -// aaf32c78-947b-439a-a180-2e144ec37792 #define NVRAM_VSS2_AUTH_VAR_KEY_DATABASE_GUID_PART1 0xaaf32c78 -const UByteArray NVRAM_VSS2_AUTH_VAR_KEY_DATABASE_GUID -("\x78\x2C\xF3\xAA\x7B\x94\x9A\x43\xA1\x80\x2E\x14\x4E\xC3\x77\x92"); +extern const UByteArray NVRAM_VSS2_AUTH_VAR_KEY_DATABASE_GUID; // AAF32C78-947B-439A-A180-2E144EC37792 #define NVRAM_VSS2_STORE_GUID_PART1 0xddcf3617 -const UByteArray NVRAM_VSS2_STORE_GUID -("\x17\x36\xCF\xDD\x75\x32\x64\x41\x98\xB6\xFE\x85\x70\x7F\xFE\x7D"); +extern const UByteArray NVRAM_VSS2_STORE_GUID; // DDCF3617-3275-4164-98B6-FE85707FFE7D -const UByteArray NVRAM_FDC_STORE_GUID -("\x16\x36\xCF\xDD\x75\x32\x64\x41\x98\xB6\xFE\x85\x70\x7F\xFE\x7D"); +extern const UByteArray NVRAM_FDC_STORE_GUID; // DDCF3616-3275-4164-98B6-FE85707FFE7D // Variable store header typedef struct VSS2_VARIABLE_STORE_HEADER_ { @@ -226,14 +203,8 @@ typedef struct FDC_VOLUME_HEADER_ { // #define EFI_FAULT_TOLERANT_WORKING_BLOCK_VALID 0x1 #define EFI_FAULT_TOLERANT_WORKING_BLOCK_INVALID 0x2 - -// 9E58292B-7C68-497D-0ACE6500FD9F1B95 -const UByteArray EDKII_WORKING_BLOCK_SIGNATURE_GUID -("\x2B\x29\x58\x9E\x68\x7C\x7D\x49\x0A\xCE\x65\x00\xFD\x9F\x1B\x95", 16); - -// 9E58292B-7C68-497D-A0CE6500FD9F1B95 -const UByteArray VSS2_WORKING_BLOCK_SIGNATURE_GUID -("\x2B\x29\x58\x9E\x68\x7C\x7D\x49\xA0\xCE\x65\x00\xFD\x9F\x1B\x95", 16); +extern const UByteArray EDKII_WORKING_BLOCK_SIGNATURE_GUID; // 9E58292B-7C68-497D-0ACE6500FD9F1B95 +extern const UByteArray VSS2_WORKING_BLOCK_SIGNATURE_GUID; // 9E58292B-7C68-497D-A0CE6500FD9F1B95 #define NVRAM_MAIN_STORE_VOLUME_GUID_DATA1 0xFFF12B8D #define EDKII_WORKING_BLOCK_SIGNATURE_GUID_DATA1 0x9E58292B @@ -355,8 +326,7 @@ extern UString evsaAttributesToUString(const UINT32 attributes); #define NVRAM_PHOENIX_FLASH_MAP_SIGNATURE_LENGTH 10 // _FLASH_MAP -const UByteArray NVRAM_PHOENIX_FLASH_MAP_SIGNATURE -("\x5F\x46\x4C\x41\x53\x48\x5F\x4D\x41\x50", 10); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_SIGNATURE; typedef struct PHOENIX_FLASH_MAP_HEADER_ { UINT8 Signature[10]; // _FLASH_MAP signature @@ -378,70 +348,25 @@ typedef struct PHOENIX_FLASH_MAP_ENTRY_ { extern UString flashMapGuidToUString(const EFI_GUID & guid); -// B091E7D2-05A0-4198-94F0-74B7B8C55459 -const UByteArray NVRAM_PHOENIX_FLASH_MAP_VOLUME_HEADER -("\xD2\xE7\x91\xB0\xA0\x05\x98\x41\x94\xF0\x74\xB7\xB8\xC5\x54\x59", 16); - -// FD3F690E-B4B0-4D68-89DB-19A1A3318F90 -const UByteArray NVRAM_PHOENIX_FLASH_MAP_MICROCODES_GUID -("\x0E\x69\x3F\xFD\xB0\xB4\x68\x4D\x89\xDB\x19\xA1\xA3\x31\x8F\x90", 16); - -// 46310243-7B03-4132-BE44-2243FACA7CDD -const UByteArray NVRAM_PHOENIX_FLASH_MAP_CMDB_GUID -("\x43\x02\x31\x46\x03\x7B\x32\x41\xBE\x44\x22\x43\xFA\xCA\x7C\xDD", 16); - -// 1B2C4952-D778-4B64-BDA1-15A36F5FA545 -const UByteArray NVRAM_PHOENIX_FLASH_MAP_PUBKEY1_GUID -("\x52\x49\x2C\x1B\x78\xD7\x64\x4B\xBD\xA1\x15\xA3\x6F\x5F\xA5\x45", 16); - -// 127C1C4E-9135-46E3-B006-F9808B0559A5 -const UByteArray NVRAM_PHOENIX_FLASH_MAP_MARKER1_GUID -("\x4E\x1C\x7C\x12\x35\x91\xE3\x46\xB0\x06\xF9\x80\x8B\x05\x59\xA5", 16); - -// 7CE75114-8272-45AF-B536-761BD38852CE -const UByteArray NVRAM_PHOENIX_FLASH_MAP_PUBKEY2_GUID -("\x14\x51\xE7\x7C\x72\x82\xAF\x45\xB5\x36\x76\x1B\xD3\x88\x52\xCE", 16); - -// 071A3DBE-CFF4-4B73-83F0-598C13DCFDD5 -const UByteArray NVRAM_PHOENIX_FLASH_MAP_MARKER2_GUID -("\xBE\x3D\x1A\x07\xF4\xCF\x73\x4B\x83\xF0\x59\x8C\x13\xDC\xFD\xD5", 16); - -// FACFB110-7BFD-4EFB-873E-88B6B23B97EA -const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA1_GUID -("\x10\xB1\xCF\xFA\xFD\x7B\xFB\x4E\x87\x3E\x88\xB6\xB2\x3B\x97\xEA", 16); - -// E68DC11A-A5F4-4AC3-AA2E-29E298BFF645 -const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA2_GUID -("\x1A\xC1\x8D\xE6\xF4\xA5\xC3\x4A\xAA\x2E\x29\xE2\x98\xBF\xF6\x45", 16); - -// 4B3828AE-0ACE-45B6-8CDB-DAFC28BBF8C5 -const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA3_GUID -("\xAE\x28\x38\x4B\xCE\x0A\xB6\x45\x8C\xDB\xDA\xFC\x28\xBB\xF8\xC5", 16); - -// C22E6B8A-8159-49A3-B353-E84B79DF19C0 -const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA4_GUID -("\x8A\x6B\x2E\xC2\x59\x81\xA3\x49\xB3\x53\xE8\x4B\x79\xDF\x19\xC0", 16); - -// B6B5FAB9-75C4-4AAE-8314-7FFFA7156EAA -const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA5_GUID -("\xB9\xFA\xB5\xB6\xC4\x75\xAE\x4A\x83\x14\x7F\xFF\xA7\x15\x6E\xAA", 16); - -// 919B9699-8DD0-4376-AA0B-0E54CCA47D8F -const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA6_GUID -("\x99\x96\x9B\x91\xD0\x8D\x76\x43\xAA\x0B\x0E\x54\xCC\xA4\x7D\x8F", 16); - -// 58A90A52-929F-44F8-AC35-A7E1AB18AC91 -const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA7_GUID -("\x52\x0A\xA9\x58\x9F\x92\xF8\x44\xAC\x35\xA7\xE1\xAB\x18\xAC\x91", 16); - -// 8CB71915-531F-4AF5-82BF-A09140817BAA -const UByteArray NVRAM_PHOENIX_FLASH_MAP_SELF_GUID -("\x15\x19\xB7\x8C\x1F\x53\xF5\x4A\x82\xBF\xA0\x91\x40\x81\x7B\xAA", 16); +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_VOLUME_HEADER; // B091E7D2-05A0-4198-94F0-74B7B8C55459 +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_MICROCODES_GUID; // FD3F690E-B4B0-4D68-89DB-19A1A3318F90 +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_CMDB_GUID; // 46310243-7B03-4132-BE44-2243FACA7CDD +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_PUBKEY1_GUID; // 1B2C4952-D778-4B64-BDA1-15A36F5FA545 +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_MARKER1_GUID; // 127C1C4E-9135-46E3-B006-F9808B0559A5 +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_PUBKEY2_GUID; // 7CE75114-8272-45AF-B536-761BD38852CE +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_MARKER2_GUID; // 071A3DBE-CFF4-4B73-83F0-598C13DCFDD5 +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA1_GUID; // FACFB110-7BFD-4EFB-873E-88B6B23B97EA +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA2_GUID; // E68DC11A-A5F4-4AC3-AA2E-29E298BFF645 +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA3_GUID; // 4B3828AE-0ACE-45B6-8CDB-DAFC28BBF8C5 +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA4_GUID; // C22E6B8A-8159-49A3-B353-E84B79DF19C0 +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA5_GUID; // B6B5FAB9-75C4-4AAE-8314-7FFFA7156EAA +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA6_GUID; // 919B9699-8DD0-4376-AA0B-0E54CCA47D8F +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_EVSA7_GUID; // 58A90A52-929F-44F8-AC35-A7E1AB18AC91 +extern const UByteArray NVRAM_PHOENIX_FLASH_MAP_SELF_GUID; // 8CB71915-531F-4AF5-82BF-A09140817BAA // // SLIC pubkey and marker // - typedef struct OEM_ACTIVATION_PUBKEY_ { UINT32 Type; // 0 UINT32 Size; // 0x9C @@ -478,7 +403,6 @@ typedef struct OEM_ACTIVATION_MARKER_ { // // Phoenix CMDB, no londer used, requires no parsing // - typedef struct PHOENIX_CMDB_HEADER_ { UINT32 Signature; // CMDB signature UINT32 HeaderSize; // Size of this header diff --git a/common/nvramparser.cpp b/common/nvramparser.cpp index f96cbd3..aa05115 100755 --- a/common/nvramparser.cpp +++ b/common/nvramparser.cpp @@ -12,8 +12,6 @@ */ -//TODO: relax fixed restrictions once NVRAM builder is ready - #include #include "nvramparser.h" @@ -21,8 +19,7 @@ #include "utility.h" #include "nvram.h" #include "ffs.h" -#include "fit.h" -#include "uinttypes.h" +#include "intel_microcode.h" #ifdef U_ENABLE_NVRAM_PARSING_SUPPORT USTATUS NvramParser::parseNvarStore(const UModelIndex & index) @@ -308,7 +305,7 @@ USTATUS NvramParser::parseNvarStore(const UModelIndex & index) pdata.isValid = FALSE; } else // Add GUID info for valid entries - info += UString("Variable GUID: ") + guid + UString("\n"); + info += UString("Variable GUID: ") + guid + "\n"; // Add GUID index information if (hasGuidIndex) @@ -537,6 +534,8 @@ USTATUS NvramParser::findNextStore(const UModelIndex & index, const UByteArray & return U_STORES_NOT_FOUND; // TODO: add checks for restSize + // TODO: remove misaligned access by doing the signature checks differently, the current way is UB is C++ + // TODO: rewrite this all as Kaitai-based parser UINT32 offset = storeOffset; for (; offset < dataSize - sizeof(UINT32); offset++) { const UINT32* currentPos = (const UINT32*)(volume.constData() + offset); @@ -1504,7 +1503,7 @@ USTATUS NvramParser::parseVssStoreBody(const UModelIndex & index, UINT8 alignmen } else { // Add GUID and text for valid variables name = guidToUString(readUnaligned(variableGuid)); - info += UString("Variable GUID: ") + guidToUString(readUnaligned(variableGuid), false) + UString("\n"); + info += UString("Variable GUID: ") + guidToUString(readUnaligned(variableGuid), false) + "\n"; #if QT_VERSION_MAJOR >= 6 text = UString::fromUtf16((char16_t *)variableName); @@ -1859,7 +1858,7 @@ USTATUS NvramParser::parseEvsaStoreBody(const UModelIndex & index) model->setName(current, guid); } model->setText(current, name); - model->addInfo(current, UString("GUID: ") + guid + UString("\nName: ") + name + UString("\n"), false); + model->addInfo(current, UString("GUID: ") + guid + UString("\nName: ") + name + "\n", false); } } } @@ -1910,10 +1909,10 @@ USTATUS NvramParser::parseFlashMapBody(const UModelIndex & index) // Add info UString info = UString("Entry GUID: ") + guidToUString(entryHeader->Guid, false) + usprintf("\nFull size: 24h (36)\nHeader size: 24h (36)\nBody size: 0h (0)\n" - "Entry type: %04Xh\nData type: %04Xh\nMemory address: %08llXh\nSize: %08Xh\nOffset: %08Xh", + "Entry type: %04Xh\nData type: %04Xh\nMemory address: %08Xh\nSize: %08Xh\nOffset: %08Xh", entryHeader->EntryType, entryHeader->DataType, - (unsigned long long)entryHeader->PhysicalAddress, + (UINT32)entryHeader->PhysicalAddress, entryHeader->Size, entryHeader->Offset); diff --git a/common/peimage.cpp b/common/peimage.cpp index b2d39c8..7feb30b 100644 --- a/common/peimage.cpp +++ b/common/peimage.cpp @@ -30,6 +30,6 @@ UString machineTypeToUString(UINT16 machineType) case EFI_IMAGE_FILE_MACHINE_RISCV32: return UString("RISC-V 32-bit"); case EFI_IMAGE_FILE_MACHINE_RISCV64: return UString("RISC-V 64-bit"); case EFI_IMAGE_FILE_MACHINE_RISCV128: return UString("RISC-V 128-bit"); - default: return usprintf("Unknown (%04Xh)", machineType); } + return usprintf("Unknown %04Xh", machineType); } diff --git a/common/sha256.c b/common/sha256.c deleted file mode 100644 index 34a177f..0000000 --- a/common/sha256.c +++ /dev/null @@ -1,212 +0,0 @@ -/* sha256.c - -Copyright (c) 2017, LongSoft. All rights reserved. -This program and the accompanying materials -are licensed and made available under the terms and conditions of the BSD License -which accompanies this distribution. The full text of the license may be found at -http://opensource.org/licenses/bsd-license.php - -THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, -WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. - -*/ - -#include "sha256.h" -#include -#include - -struct sha256_state { - uint64_t length; - uint32_t state[8], curlen; - uint8_t buf[SHA256_DIGEST_SIZE*2]; -}; - -void sha256_init(struct sha256_state *md); -int sha256_process(struct sha256_state *md, const unsigned char *in, unsigned long inlen); -int sha256_done(struct sha256_state *md, uint8_t *out); - -#define GET_BE32(a) ((((uint32_t) (a)[0]) << 24) | (((uint32_t) (a)[1]) << 16) | \ - (((uint32_t) (a)[2]) << 8) | ((uint32_t) (a)[3])) - -#define PUT_BE32(a, val) \ - do { \ - (a)[0] = (uint8_t) ((((uint32_t) (val)) >> 24) & 0xff); \ - (a)[1] = (uint8_t) ((((uint32_t) (val)) >> 16) & 0xff); \ - (a)[2] = (uint8_t) ((((uint32_t) (val)) >> 8) & 0xff); \ - (a)[3] = (uint8_t) (((uint32_t) (val)) & 0xff); \ - } while (0) - -#define PUT_BE64(a, val) \ - do { \ - (a)[0] = (uint8_t) (((uint64_t) (val)) >> 56); \ - (a)[1] = (uint8_t) (((uint64_t) (val)) >> 48); \ - (a)[2] = (uint8_t) (((uint64_t) (val)) >> 40); \ - (a)[3] = (uint8_t) (((uint64_t) (val)) >> 32); \ - (a)[4] = (uint8_t) (((uint64_t) (val)) >> 24); \ - (a)[5] = (uint8_t) (((uint64_t) (val)) >> 16); \ - (a)[6] = (uint8_t) (((uint64_t) (val)) >> 8); \ - (a)[7] = (uint8_t) (((uint64_t) (val)) & 0xff); \ - } while (0) - -void sha256(const void *in, unsigned long inlen, void* out) -{ - struct sha256_state ctx; - sha256_init(&ctx); - sha256_process(&ctx, (const unsigned char*)in, inlen); - sha256_done(&ctx, (unsigned char *)out); -} - -/* This is based on SHA256 implementation in LibTomCrypt that was released into - * public domain by Tom St Denis. */ -/* the K array */ -static const unsigned long K[64] = { - 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, - 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL, - 0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, - 0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, - 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL, - 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL, - 0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, - 0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, - 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL, - 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL, - 0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, - 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, - 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL -}; -/* Various logical functions */ -#define RORc(x, y) \ -( ((((unsigned long) (x) & 0xFFFFFFFFUL) >> (unsigned long) ((y) & 31)) | \ -((unsigned long) (x) << (unsigned long) (32 - ((y) & 31)))) & 0xFFFFFFFFUL) -#define Ch(x,y,z) (z ^ (x & (y ^ z))) -#define Maj(x,y,z) (((x | y) & z) | (x & y)) -#define S(x, n) RORc((x), (n)) -#define R(x, n) (((x)&0xFFFFFFFFUL)>>(n)) -#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) -#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) -#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) -#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) -#ifndef MIN -#define MIN(x, y) (((x) < (y)) ? (x) : (y)) -#endif -/* compress 512-bits */ -static void sha256_compress(struct sha256_state *md, unsigned char *buf) -{ - uint32_t S[8], W[64], t0, t1; - uint32_t t; - int i; - /* copy state into S */ - for (i = 0; i < 8; i++) { - S[i] = md->state[i]; - } - /* copy the state into 512-bits into W[0..15] */ - for (i = 0; i < 16; i++) - W[i] = GET_BE32(buf + (4 * i)); - /* fill W[16..63] */ - for (i = 16; i < 64; i++) { - W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + - W[i - 16]; - } - /* Compress */ -#define RND(a,b,c,d,e,f,g,h,i) \ -t0 = (uint32_t)(h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]); \ -t1 = (uint32_t)(Sigma0(a) + Maj(a, b, c)); \ -d += t0; \ -h = t0 + t1; - for (i = 0; i < 64; ++i) { - RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], i); - t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4]; - S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t; - } - /* feedback */ - for (i = 0; i < 8; i++) { - md->state[i] = md->state[i] + S[i]; - } -} -/* Initialize the hash state */ -void sha256_init(struct sha256_state *md) -{ - md->curlen = 0; - md->length = 0; - md->state[0] = 0x6A09E667UL; - md->state[1] = 0xBB67AE85UL; - md->state[2] = 0x3C6EF372UL; - md->state[3] = 0xA54FF53AUL; - md->state[4] = 0x510E527FUL; - md->state[5] = 0x9B05688CUL; - md->state[6] = 0x1F83D9ABUL; - md->state[7] = 0x5BE0CD19UL; -} -/** - Process a block of memory though the hash - @param md The hash state - @param in The data to hash - @param inlen The length of the data (octets) - @return CRYPT_OK if successful - */ -int sha256_process(struct sha256_state *md, const unsigned char *in, - unsigned long inlen) -{ - unsigned long n; -#define block_size 64 - if (md->curlen > sizeof(md->buf)) - return -1; - while (inlen > 0) { - if (md->curlen == 0 && inlen >= block_size) { - sha256_compress(md, (unsigned char *) in); - md->length += block_size * 8; - in += block_size; - inlen -= block_size; - } else { - n = MIN(inlen, (block_size - md->curlen)); - memcpy(md->buf + md->curlen, in, n); - md->curlen += n; - in += n; - inlen -= n; - if (md->curlen == block_size) { - sha256_compress(md, md->buf); - md->length += 8 * block_size; - md->curlen = 0; - } - } - } - return 0; -} -/** - Terminate the hash to get the digest - @param md The hash state - @param out [out] The destination of the hash (32 bytes) - @return CRYPT_OK if successful - */ -int sha256_done(struct sha256_state *md, unsigned char *out) -{ - int i; - if (md->curlen >= sizeof(md->buf)) - return -1; - /* increase the length of the message */ - md->length += (uint64_t)md->curlen * 8; - /* append the '1' bit */ - md->buf[md->curlen++] = (unsigned char) 0x80; - /* if the length is currently above 56 bytes we append zeros - * then compress. Then we can fall back to padding zeros and length - * encoding like normal. - */ - if (md->curlen > 56) { - while (md->curlen < 64) { - md->buf[md->curlen++] = (unsigned char) 0; - } - sha256_compress(md, md->buf); - md->curlen = 0; - } - /* pad upto 56 bytes of zeroes */ - while (md->curlen < 56) { - md->buf[md->curlen++] = (unsigned char) 0; - } - /* store length */ - PUT_BE64(md->buf + 56, md->length); - sha256_compress(md, md->buf); - /* copy output */ - for (i = 0; i < 8; i++) - PUT_BE32(out + (4 * i), md->state[i]); - return 0; -} diff --git a/common/treemodel.cpp b/common/treemodel.cpp index 9715a79..b49c508 100644 --- a/common/treemodel.cpp +++ b/common/treemodel.cpp @@ -158,7 +158,7 @@ int TreeModel::rowCount(const UModelIndex &parent) const UINT32 TreeModel::base(const UModelIndex ¤t) const { - // TODO: rewrite this as loop if we ever see an image that is too deep for this naive implementation + // Rewrite this as loop if we ever see an image that is too deep for this naive implementation if (!current.isValid()) return 0; diff --git a/common/types.cpp b/common/types.cpp index c1fac7e..d4fcc8d 100755 --- a/common/types.cpp +++ b/common/types.cpp @@ -13,7 +13,7 @@ #include "ustring.h" #include "types.h" #include "ffs.h" -#include "fit.h" +#include "intel_fit.h" UString regionTypeToUString(const UINT8 type) { @@ -36,7 +36,7 @@ UString regionTypeToUString(const UINT8 type) case Subtypes::PttRegion: return UString("PTT"); }; - return UString("Unknown"); + return usprintf("Unknown %02Xh", type); } UString itemTypeToUString(const UINT8 type) @@ -66,7 +66,7 @@ UString itemTypeToUString(const UINT8 type) case Types::FlashMapEntry: return UString("FlashMap entry"); case Types::Microcode: return UString("Microcode"); case Types::SlicData: return UString("SLIC data"); - // ME-specific + // ME-specific case Types::FptStore: return UString("FPT store"); case Types::FptEntry: return UString("FPT entry"); case Types::IfwiHeader: return UString("IFWI header"); @@ -82,7 +82,7 @@ UString itemTypeToUString(const UINT8 type) case Types::CpdSpiEntry: return UString("CPD SPI entry"); } - return UString("Unknown"); + return usprintf("Unknown %02Xh", type); } UString itemSubtypeToUString(const UINT8 type, const UINT8 subtype) @@ -157,14 +157,14 @@ UString itemSubtypeToUString(const UINT8 type, const UINT8 subtype) if (subtype == Subtypes::GlutFptPartition) return UString("GLUT"); break; case Types::IfwiPartition: - if (subtype == Subtypes::BootIfwiPartition) return UString("Boot"); - if (subtype == Subtypes::DataIfwiPartition) return UString("Data"); + if (subtype == Subtypes::BootIfwiPartition) return UString("Boot"); + if (subtype == Subtypes::DataIfwiPartition) return UString("Data"); break; case Types::CpdPartition: - if (subtype == Subtypes::ManifestCpdPartition) return UString("Manifest"); - if (subtype == Subtypes::MetadataCpdPartition) return UString("Metadata"); - if (subtype == Subtypes::KeyCpdPartition) return UString("Key"); - if (subtype == Subtypes::CodeCpdPartition) return UString("Code"); + if (subtype == Subtypes::ManifestCpdPartition) return UString("Manifest"); + if (subtype == Subtypes::MetadataCpdPartition) return UString("Metadata"); + if (subtype == Subtypes::KeyCpdPartition) return UString("Key"); + if (subtype == Subtypes::CodeCpdPartition) return UString("Code"); break; } @@ -182,7 +182,7 @@ UString compressionTypeToUString(const UINT8 algorithm) case COMPRESSION_ALGORITHM_LZMA_INTEL_LEGACY: return UString("Intel legacy LZMA"); } - return UString("Unknown"); + return usprintf("Unknown %02Xh", algorithm); } UString actionTypeToUString(const UINT8 action) @@ -197,23 +197,41 @@ UString actionTypeToUString(const UINT8 action) case Actions::Rebase: return UString("Rebase"); } - return UString("Unknown"); + return usprintf("Unknown %02Xh", action); } UString fitEntryTypeToUString(const UINT8 type) { switch (type & 0x7F) { - case FIT_TYPE_HEADER: return UString("FIT Header"); - case FIT_TYPE_MICROCODE: return UString("Microcode"); - case FIT_TYPE_BIOS_AC_MODULE: return UString("BIOS ACM"); - case FIT_TYPE_BIOS_INIT_MODULE: return UString("BIOS Init"); - case FIT_TYPE_TPM_POLICY: return UString("TPM Policy"); - case FIT_TYPE_BIOS_POLICY_DATA: return UString("BIOS Policy Data"); - case FIT_TYPE_TXT_CONF_POLICY: return UString("TXT Configuration Policy"); - case FIT_TYPE_AC_KEY_MANIFEST: return UString("BootGuard Key Manifest"); - case FIT_TYPE_AC_BOOT_POLICY: return UString("BootGuard Boot Policy"); - case FIT_TYPE_EMPTY: return UString("Empty"); + case INTEL_FIT_TYPE_HEADER: return UString("FIT Header"); + case INTEL_FIT_TYPE_MICROCODE: return UString("Microcode"); + case INTEL_FIT_TYPE_STARTUP_AC_MODULE: return UString("Startup ACM"); + case INTEL_FIT_TYPE_DIAG_AC_MODULE: return UString("Diagnostic ACM"); + case INTEL_FIT_TYPE_BIOS_STARTUP_MODULE: return UString("BIOS Startup Module"); + case INTEL_FIT_TYPE_TPM_POLICY: return UString("TPM Policy"); + case INTEL_FIT_TYPE_BIOS_POLICY: return UString("BIOS Policy"); + case INTEL_FIT_TYPE_TXT_POLICY: return UString("TXT Policy"); + case INTEL_FIT_TYPE_BOOT_GUARD_KEY_MANIFEST: return UString("BootGuard Key Manifest"); + case INTEL_FIT_TYPE_BOOT_GUARD_BOOT_POLICY: return UString("BootGuard Boot Policy"); + case INTEL_FIT_TYPE_CSE_SECURE_BOOT: return UString("CSE SecureBoot Settings"); + case INTEL_FIT_TYPE_ACM_FEATURE_POLICY: return UString("ACM Feature Policy"); + case INTEL_FIT_TYPE_JMP_DEBUG_POLICY: return UString("JMP Debug Policy"); + case INTEL_FIT_TYPE_EMPTY: return UString("Empty"); } - return UString("Unknown"); + return usprintf("Unknown %02Xh", (type & 0x7F)); +} + +UString hashTypeToUString(const UINT16 algorithm_id) +{ + switch (algorithm_id) { + case TCG_HASH_ALGORITHM_ID_SHA1: return UString("SHA1"); + case TCG_HASH_ALGORITHM_ID_SHA256: return UString("SHA256"); + case TCG_HASH_ALGORITHM_ID_SHA384: return UString("SHA384"); + case TCG_HASH_ALGORITHM_ID_SHA512: return UString("SHA512"); + case TCG_HASH_ALGORITHM_ID_NULL: return UString("NULL"); + case TCG_HASH_ALGORITHM_ID_SM3: return UString("SM3"); + } + + return usprintf("Unknown %04Xh", algorithm_id); } diff --git a/common/types.h b/common/types.h index 3b6d7e3..bff89da 100755 --- a/common/types.h +++ b/common/types.h @@ -197,5 +197,6 @@ extern UString itemSubtypeToUString(const UINT8 type, const UINT8 subtype); extern UString compressionTypeToUString(const UINT8 algorithm); extern UString regionTypeToUString(const UINT8 type); extern UString fitEntryTypeToUString(const UINT8 type); +extern UString hashTypeToUString(const UINT16 digest_agorithm_id); #endif // TYPES_H diff --git a/common/ustring.h b/common/ustring.h index a6a55be..cdbcdf1 100644 --- a/common/ustring.h +++ b/common/ustring.h @@ -13,6 +13,8 @@ WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. #ifndef USTRING_H #define USTRING_H +#include "basetypes.h" + #if defined (QT_CORE_LIB) // Use Qt class, if Qt is available #include @@ -24,7 +26,6 @@ WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. #include "bstrlib/bstrwrap.h" #define UString CBString #endif // QT_CORE_LIB -#include "uinttypes.h" UString usprintf(const char* fmt, ...) ATTRIBUTE_FORMAT_(printf, 1, 2); UString urepeated(char c, int len); diff --git a/common/utility.cpp b/common/utility.cpp index b3216a9..f487e6c 100755 --- a/common/utility.cpp +++ b/common/utility.cpp @@ -359,6 +359,7 @@ USTATUS decompress(const UByteArray & compressedData, const UINT8 compressionTyp return U_CUSTOMIZED_DECOMPRESSION_FAILED; } + // TODO: need to correctly handle non-x86 architecture of the FW image // After LZMA decompression, the data need to be converted to the raw data. UINT32 state = 0; const UINT8 x86LookAhead = 4; diff --git a/kaitai_regenerate.sh b/kaitai_regenerate.sh new file mode 100755 index 0000000..8c91ba6 --- /dev/null +++ b/kaitai_regenerate.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +UTARGET=$(uname) + +# Determine platform +if [ "$UTARGET" = "Darwin" ]; then + export UPLATFORM="mac" +elif [ "$UTARGET" = "Linux" ]; then + export UPLATFORM="linux_$(uname -m)" +elif [ "${UTARGET/MINGW32/}" != "$UTARGET" ]; then + export UPLATFORM="win32" +else + # Fallback to something... + export UPLATFORM="$UTARGET" +fi + +# Generate +echo "Attempting to to generate parsers from Kaitai KSY files on ${UPLATFORM}..." +kaitai-struct-compiler --target cpp_stl --outdir common/generated common/ksy/* || exit 1 + +# Show generated files +find -E common/generated \ + -regex '.*\.(cpp|h)' \ + -print || exit 1 + +# Replace global includes for kaitai with local ones (<> -> "") +find -E common/generated \ + -regex '.*\.(cpp|h)' \ + -exec sed -i '' '/^#include ]/\"/g' {} + || exit 1 + +# Add .. to the include path for kaitai includes +find -E common/generated \ + -regex '.*\.(cpp|h)' \ + -exec sed -i '' '/^#include \"kaitai\//s/kaitai\//..\/kaitai\//g' {} + || exit 1 + +# Suppress "p__root - unused parameter" warning +find -E common/generated \ + -regex '.*\.(cpp)' \ + -exec sed -i '' '/^ m__root = this;/s/;/; (void)p__root;/g' {} + || exit 1 + +exit 0