diff options
249 files changed, 5344 insertions, 2430 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index e20d770..1fcd8f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,7 +285,8 @@ if(CMake_RUN_IWYU) message(FATAL_ERROR "CMake_RUN_IWYU is ON but include-what-you-use is not found!") endif() set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE - "${IWYU_COMMAND};-Xiwyu;--mapping_file=${CMake_SOURCE_DIR}/Utilities/IWYU/mapping.imp;-w;-DCMAKE_IWYU") + "${IWYU_COMMAND};-Xiwyu;--mapping_file=${CMake_SOURCE_DIR}/Utilities/IWYU/mapping.imp;-w") + list(APPEND CMAKE_CXX_INCLUDE_WHAT_YOU_USE ${CMake_IWYU_OPTIONS}) endif() diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 01987be..84f6abb 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -36,7 +36,9 @@ To contribute patches: #. Push a topic branch to a personal repository fork on GitLab. #. Create a GitLab Merge Request targeting the upstream ``master`` branch (even if the change is intended for merge to the ``release`` branch). - Check the box to allow edits from maintainers. + Check the box labelled "Allow commits from members who can merge to the + target branch". This will allow maintainers to make minor edits on your + behalf. The merge request will enter the `CMake Review Process`_ for consideration. @@ -47,6 +49,20 @@ The merge request will enter the `CMake Review Process`_ for consideration. .. _`commit messages`: Help/dev/review.rst#commit-messages .. _`CMake Review Process`: Help/dev/review.rst +CMake Dashboard Client +====================== + +The *integration testing* step of the `CMake Review Process`_ uses a set of +testing machines that follow an integration branch on their own schedule to +drive testing and submit results to the `CMake CDash Page`_. Anyone is +welcome to provide testing machines in order to help keep support for their +platforms working. + +See documentation on `CMake Testing Process`_ for more information. + +.. _`CMake CDash Page`: https://open.cdash.org/index.php?project=CMake +.. _`CMake Testing Process`: Help/dev/testing.rst + License ======= diff --git a/CTestCustom.cmake.in b/CTestCustom.cmake.in index eb8b630..aef375a 100644 --- a/CTestCustom.cmake.in +++ b/CTestCustom.cmake.in @@ -92,6 +92,7 @@ list(APPEND CTEST_CUSTOM_WARNING_EXCEPTION "cmFortranLexer.cxx:[0-9]+:[0-9]+: warning: Call to 'realloc' has an allocation size of 0 bytes" "testProcess.*warning: Dereference of null pointer .loaded from variable .invalidAddress.." "liblzma/simple/x86.c:[0-9]+:[0-9]+: warning: The result of the '<<' expression is undefined" + "liblzma/common/index_encoder.c:[0-9]+:[0-9]+: warning: Value stored to .* during its initialization is never read" "libuv/src/.*:[0-9]+:[0-9]+: warning: Dereference of null pointer" "libuv/src/.*:[0-9]+:[0-9]+: warning: The left operand of '==' is a garbage value" ) diff --git a/Help/dev/testing.rst b/Help/dev/testing.rst index 1b29acf..23d0ca3 100644 --- a/Help/dev/testing.rst +++ b/Help/dev/testing.rst @@ -26,13 +26,14 @@ commands to set up a new integration testing client: $ git clone https://gitlab.kitware.com/cmake/dashboard-scripts.git CMakeScripts $ cd CMakeScripts -The ``cmake_common.cmake`` script contains comments at the top with +The `cmake_common.cmake`_ script contains comments at the top with instructions to set up a testing client. As it instructs, create a CTest script with local settings and include ``cmake_common.cmake``. .. _`CMake Review Process`: review.rst .. _`CMake CDash Page`: https://open.cdash.org/index.php?project=CMake .. _`CMake Dashboard Scripts Repository`: https://gitlab.kitware.com/cmake/dashboard-scripts +.. _`cmake_common.cmake`: https://gitlab.kitware.com/cmake/dashboard-scripts/blob/master/cmake_common.cmake Nightly Start Time ------------------ diff --git a/Help/generator/Visual Studio 10 2010.rst b/Help/generator/Visual Studio 10 2010.rst index 04eef10..b8d6872 100644 --- a/Help/generator/Visual Studio 10 2010.rst +++ b/Help/generator/Visual Studio 10 2010.rst @@ -18,6 +18,12 @@ a target platform name optionally at the end of this generator name: For compatibility with CMake versions prior to 3.0, one may specify this generator using the name ``Visual Studio 10`` without the year component. +Project Types +^^^^^^^^^^^^^ + +Only Visual C++ and C# projects may be generated. Other types of +projects (Database, Website, etc.) are not supported. + Toolset Selection ^^^^^^^^^^^^^^^^^ diff --git a/Help/generator/Visual Studio 11 2012.rst b/Help/generator/Visual Studio 11 2012.rst index 347a153..8e9998a 100644 --- a/Help/generator/Visual Studio 11 2012.rst +++ b/Help/generator/Visual Studio 11 2012.rst @@ -21,6 +21,12 @@ a target platform name optionally at the end of this generator name: For compatibility with CMake versions prior to 3.0, one may specify this generator using the name "Visual Studio 11" without the year component. +Project Types +^^^^^^^^^^^^^ + +Only Visual C++ and C# projects may be generated. Other types of +projects (JavaScript, Database, Website, etc.) are not supported. + Toolset Selection ^^^^^^^^^^^^^^^^^ diff --git a/Help/generator/Visual Studio 12 2013.rst b/Help/generator/Visual Studio 12 2013.rst index 5071b20..03f7586 100644 --- a/Help/generator/Visual Studio 12 2013.rst +++ b/Help/generator/Visual Studio 12 2013.rst @@ -18,6 +18,12 @@ a target platform name optionally at the end of this generator name: For compatibility with CMake versions prior to 3.0, one may specify this generator using the name "Visual Studio 12" without the year component. +Project Types +^^^^^^^^^^^^^ + +Only Visual C++ and C# projects may be generated. Other types of +projects (JavaScript, Powershell, Python, etc.) are not supported. + Toolset Selection ^^^^^^^^^^^^^^^^^ diff --git a/Help/generator/Visual Studio 14 2015.rst b/Help/generator/Visual Studio 14 2015.rst index 64254b5..e55bc73 100644 --- a/Help/generator/Visual Studio 14 2015.rst +++ b/Help/generator/Visual Studio 14 2015.rst @@ -15,6 +15,12 @@ a target platform name optionally at the end of this generator name: ``Visual Studio 14 2015 ARM`` Specify target platform ``ARM``. +Project Types +^^^^^^^^^^^^^ + +Only Visual C++ and C# projects may be generated. Other types of +projects (JavaScript, Powershell, Python, etc.) are not supported. + Toolset Selection ^^^^^^^^^^^^^^^^^ diff --git a/Help/generator/Visual Studio 15 2017.rst b/Help/generator/Visual Studio 15 2017.rst index 2cf1aa0..809be4b 100644 --- a/Help/generator/Visual Studio 15 2017.rst +++ b/Help/generator/Visual Studio 15 2017.rst @@ -15,6 +15,12 @@ a target platform name optionally at the end of this generator name: ``Visual Studio 15 2017 ARM`` Specify target platform ``ARM``. +Project Types +^^^^^^^^^^^^^ + +Only Visual C++ and C# projects may be generated. Other types of +projects (JavaScript, Powershell, Python, etc.) are not supported. + Instance Selection ^^^^^^^^^^^^^^^^^^ diff --git a/Help/manual/cmake-language.7.rst b/Help/manual/cmake-language.7.rst index 87f8f9d..591f73d 100644 --- a/Help/manual/cmake-language.7.rst +++ b/Help/manual/cmake-language.7.rst @@ -399,6 +399,11 @@ and how their values are set. An *environment variable reference* has the form ``$ENV{VAR}`` and is evaluated in the same contexts as a normal variable reference. +See :variable:`ENV` for more information. + +A *cache variable reference* has the form ``$CACHE{VAR}`` and +is evaluated in the same contexts as a normal variable reference. +See :variable:`CACHE` for more information. Comments -------- @@ -543,6 +548,8 @@ to the binding in the current directory scope, if any. If a is found, or no binding is found, CMake then searches for a cache entry. If a cache entry is found, its value is used. Otherwise, the variable reference evaluates to an empty string. +The ``$CACHE{VAR}`` syntax can be used to do direct cache entry +lookups. The :manual:`cmake-variables(7)` manual documents many variables that are provided by CMake or have meaning to CMake when set diff --git a/Help/manual/cmake-properties.7.rst b/Help/manual/cmake-properties.7.rst index 274206f..c9a38fc 100644 --- a/Help/manual/cmake-properties.7.rst +++ b/Help/manual/cmake-properties.7.rst @@ -339,6 +339,24 @@ Properties on Targets /prop_tgt/XCODE_ATTRIBUTE_an-attribute /prop_tgt/XCODE_EXPLICIT_FILE_TYPE /prop_tgt/XCODE_PRODUCT_TYPE + /prop_tgt/XCODE_SCHEME_ADDRESS_SANITIZER + /prop_tgt/XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN + /prop_tgt/XCODE_SCHEME_THREAD_SANITIZER + /prop_tgt/XCODE_SCHEME_THREAD_SANITIZER_STOP + /prop_tgt/XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER + /prop_tgt/XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP + /prop_tgt/XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER + /prop_tgt/XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP + /prop_tgt/XCODE_SCHEME_MALLOC_SCRIBBLE + /prop_tgt/XCODE_SCHEME_MALLOC_GUARD_EDGES + /prop_tgt/XCODE_SCHEME_GUARD_MALLOC + /prop_tgt/XCODE_SCHEME_ZOMBIE_OBJECTS + /prop_tgt/XCODE_SCHEME_MALLOC_STACK + /prop_tgt/XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE + /prop_tgt/XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS + /prop_tgt/XCODE_SCHEME_EXECUTABLE + /prop_tgt/XCODE_SCHEME_ARGUMENTS + /prop_tgt/XCODE_SCHEME_ENVIRONMENT /prop_tgt/XCTEST .. _`Test Properties`: diff --git a/Help/manual/cmake-variables.7.rst b/Help/manual/cmake-variables.7.rst index 48b1f20..4ddbfe5 100644 --- a/Help/manual/cmake-variables.7.rst +++ b/Help/manual/cmake-variables.7.rst @@ -204,6 +204,21 @@ Variables that Change Behavior /variable/CMAKE_WARN_DEPRECATED /variable/CMAKE_WARN_ON_ABSOLUTE_INSTALL_DESTINATION /variable/CMAKE_XCODE_GENERATE_TOP_LEVEL_PROJECT_ONLY + /variable/CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER + /variable/CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN + /variable/CMAKE_XCODE_SCHEME_THREAD_SANITIZER + /variable/CMAKE_XCODE_SCHEME_THREAD_SANITIZER_STOP + /variable/CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER + /variable/CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP + /variable/CMAKE_XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER + /variable/CMAKE_XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP + /variable/CMAKE_XCODE_SCHEME_MALLOC_SCRIBBLE + /variable/CMAKE_XCODE_SCHEME_MALLOC_GUARD_EDGES + /variable/CMAKE_XCODE_SCHEME_GUARD_MALLOC + /variable/CMAKE_XCODE_SCHEME_ZOMBIE_OBJECTS + /variable/CMAKE_XCODE_SCHEME_MALLOC_STACK + /variable/CMAKE_XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE + /variable/CMAKE_XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS /variable/PackageName_ROOT Variables that Describe the System @@ -215,6 +230,7 @@ Variables that Describe the System /variable/ANDROID /variable/APPLE /variable/BORLAND + /variable/CACHE /variable/CMAKE_CL_64 /variable/CMAKE_COMPILER_2005 /variable/CMAKE_HOST_APPLE diff --git a/Help/prop_tgt/XCODE_SCHEME_ADDRESS_SANITIZER.rst b/Help/prop_tgt/XCODE_SCHEME_ADDRESS_SANITIZER.rst new file mode 100644 index 0000000..694cd77 --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_ADDRESS_SANITIZER.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_ADDRESS_SANITIZER +------------------------------ + +Whether to enable ``Address Sanitizer`` in the Diagnostics +section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER` if it is set +when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN.rst b/Help/prop_tgt/XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN.rst new file mode 100644 index 0000000..2803da0 --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN +----------------------------------------------- + +Whether to enable ``Detect use of stack after return`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN` +if it is set when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_ARGUMENTS.rst b/Help/prop_tgt/XCODE_SCHEME_ARGUMENTS.rst new file mode 100644 index 0000000..2eac4a9 --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_ARGUMENTS.rst @@ -0,0 +1,10 @@ +XCODE_SCHEME_ARGUMENTS +---------------------- + +Specify command line arguments that should be added to the Arguments +section of the generated Xcode scheme. + +If set to a list of arguments those will be added to the scheme. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER.rst b/Help/prop_tgt/XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER.rst new file mode 100644 index 0000000..75fc326 --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER +---------------------------------------- + +Whether to disable the ``Main Thread Checker`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER` +if it is set when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS.rst b/Help/prop_tgt/XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS.rst new file mode 100644 index 0000000..a7fab66 --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS +---------------------------------- + +Whether to enable ``Dynamic Library Loads`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS` if it is set +when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE.rst b/Help/prop_tgt/XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE.rst new file mode 100644 index 0000000..162fc45 --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE +------------------------------------- + +Whether to enable ``Dynamic Linker API usage`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE` if it is set +when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_ENVIRONMENT.rst b/Help/prop_tgt/XCODE_SCHEME_ENVIRONMENT.rst new file mode 100644 index 0000000..1dbd6c4 --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_ENVIRONMENT.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_ENVIRONMENT +------------------------ + +Specify environment variables that should be added to the Arguments +section of the generated Xcode scheme. + +If set to a list of environment variables and values of the form +``MYVAR=value`` those environment variables will be added to the +scheme. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_EXECUTABLE.rst b/Help/prop_tgt/XCODE_SCHEME_EXECUTABLE.rst new file mode 100644 index 0000000..d0427e2 --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_EXECUTABLE.rst @@ -0,0 +1,9 @@ +XCODE_SCHEME_EXECUTABLE +----------------------- + +Specify path to executable in the Info section of the generated +Xcode scheme. If not set the schema generator will select the +current target if it is actually executable. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_GUARD_MALLOC.rst b/Help/prop_tgt/XCODE_SCHEME_GUARD_MALLOC.rst new file mode 100644 index 0000000..64e1990 --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_GUARD_MALLOC.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_GUARD_MALLOC +------------------------------ + +Whether to enable ``Guard Malloc`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_GUARD_MALLOC` if it is set +when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP.rst b/Help/prop_tgt/XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP.rst new file mode 100644 index 0000000..99c112f --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP.rst @@ -0,0 +1,13 @@ +XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP +------------------------------------- + +Whether to enable the ``Main Thread Checker`` option +``Pause on issues`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP` if it is set +when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_MALLOC_GUARD_EDGES.rst b/Help/prop_tgt/XCODE_SCHEME_MALLOC_GUARD_EDGES.rst new file mode 100644 index 0000000..ef3852a --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_MALLOC_GUARD_EDGES.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_MALLOC_GUARD_EDGES +------------------------------- + +Whether to enable ``Malloc Guard Edges`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_MALLOC_GUARD_EDGES` if it is set +when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_MALLOC_SCRIBBLE.rst b/Help/prop_tgt/XCODE_SCHEME_MALLOC_SCRIBBLE.rst new file mode 100644 index 0000000..75baba2 --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_MALLOC_SCRIBBLE.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_MALLOC_SCRIBBLE +------------------------------ + +Whether to enable ``Malloc Scribble`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_MALLOC_SCRIBBLE` if it is set +when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_MALLOC_STACK.rst b/Help/prop_tgt/XCODE_SCHEME_MALLOC_STACK.rst new file mode 100644 index 0000000..984022c --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_MALLOC_STACK.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_MALLOC_STACK +------------------------- + +Whether to enable ``Malloc Stack`` in the Diagnostics +section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_MALLOC_STACK` if it is set +when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_THREAD_SANITIZER.rst b/Help/prop_tgt/XCODE_SCHEME_THREAD_SANITIZER.rst new file mode 100644 index 0000000..825ac5b --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_THREAD_SANITIZER.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_THREAD_SANITIZER +----------------------------- + +Whether to enable ``Thread Sanitizer`` in the Diagnostics +section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_THREAD_SANITIZER` if it is set +when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_THREAD_SANITIZER_STOP.rst b/Help/prop_tgt/XCODE_SCHEME_THREAD_SANITIZER_STOP.rst new file mode 100644 index 0000000..86f894e --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_THREAD_SANITIZER_STOP.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_THREAD_SANITIZER_STOP +---------------------------------- + +Whether to enable ``Thread Sanitizer - Pause on issues`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_THREAD_SANITIZER_STOP` if it is set +when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER.rst b/Help/prop_tgt/XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER.rst new file mode 100644 index 0000000..829a62e --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER +------------------------------------------ + +Whether to enable ``Undefined Behavior Sanitizer`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER` +if it is set when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP.rst b/Help/prop_tgt/XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP.rst new file mode 100644 index 0000000..5e382ca --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP.rst @@ -0,0 +1,13 @@ +XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP +----------------------------------------------- + +Whether to enable ``Undefined Behavior Sanitizer`` option +``Pause on issues`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP` +if it is set when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/prop_tgt/XCODE_SCHEME_ZOMBIE_OBJECTS.rst b/Help/prop_tgt/XCODE_SCHEME_ZOMBIE_OBJECTS.rst new file mode 100644 index 0000000..80b954a --- /dev/null +++ b/Help/prop_tgt/XCODE_SCHEME_ZOMBIE_OBJECTS.rst @@ -0,0 +1,12 @@ +XCODE_SCHEME_ZOMBIE_OBJECTS +------------------------------ + +Whether to enable ``Zombie Objects`` +in the Diagnostics section of the generated Xcode scheme. + +This property is initialized by the value of the variable +:variable:`CMAKE_XCODE_SCHEME_ZOMBIE_OBJECTS` if it is set +when a target is created. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/release/dev/FindMatlab-mcc.rst b/Help/release/dev/FindMatlab-mcc.rst new file mode 100644 index 0000000..71387f3 --- /dev/null +++ b/Help/release/dev/FindMatlab-mcc.rst @@ -0,0 +1,5 @@ +FindMatlab-mcc +-------------- + +* The :module:`FindMatlab` module gained a new ``MCC_COMPILER`` + component to request finding the Matlab Compiler add-on. diff --git a/Help/release/dev/xcode_schemes_config.rst b/Help/release/dev/xcode_schemes_config.rst new file mode 100644 index 0000000..f9b5314 --- /dev/null +++ b/Help/release/dev/xcode_schemes_config.rst @@ -0,0 +1,5 @@ +xcode_schemes_config +-------------------- + +* The :generator:`Xcode` generator learned to configure more Xcode Scheme + fields. See the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable. diff --git a/Help/variable/CACHE.rst b/Help/variable/CACHE.rst new file mode 100644 index 0000000..230739a --- /dev/null +++ b/Help/variable/CACHE.rst @@ -0,0 +1,17 @@ +CACHE +----- + +Read cache variables. + +Use the syntax ``$CACHE{VAR}`` to read cache entry ``VAR``. +See the :ref:`cmake-language(7) variables <CMake Language Variables>` +documentation for more complete documentation of the interaction of +normal variables and cache entries. + +When evaluating :ref:`Variable References` of the form ``${VAR}``, +CMake first searches for a normal variable with that name, and if not +found CMake will search for a cache entry with that name. +The ``$CACHE{VAR}`` syntax can be used to do direct cache lookup and +ignore any existing normal variable. + +See the :command:`set` command to see how to write cache variables. diff --git a/Help/variable/CMAKE_XCODE_GENERATE_SCHEME.rst b/Help/variable/CMAKE_XCODE_GENERATE_SCHEME.rst index c9fcc92..ed20bbe 100644 --- a/Help/variable/CMAKE_XCODE_GENERATE_SCHEME.rst +++ b/Help/variable/CMAKE_XCODE_GENERATE_SCHEME.rst @@ -9,3 +9,31 @@ actions from the command line. The Xcode Schema Generator is still experimental and subject to change. + +The following target properties overwrite the default of the +corresponding settings on the "Diagnostic" tab for each schema file. +Each of those is initialized by the respective ``CMAKE_`` variable +at target creation time. + +- :prop_tgt:`XCODE_SCHEME_ADDRESS_SANITIZER` +- :prop_tgt:`XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN` +- :prop_tgt:`XCODE_SCHEME_THREAD_SANITIZER` +- :prop_tgt:`XCODE_SCHEME_THREAD_SANITIZER_STOP` +- :prop_tgt:`XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER` +- :prop_tgt:`XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP` +- :prop_tgt:`XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER` +- :prop_tgt:`XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP` +- :prop_tgt:`XCODE_SCHEME_MALLOC_SCRIBBLE` +- :prop_tgt:`XCODE_SCHEME_MALLOC_GUARD_EDGES` +- :prop_tgt:`XCODE_SCHEME_GUARD_MALLOC` +- :prop_tgt:`XCODE_SCHEME_ZOMBIE_OBJECTS` +- :prop_tgt:`XCODE_SCHEME_MALLOC_STACK` +- :prop_tgt:`XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE` +- :prop_tgt:`XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS` + +The following target properties will be applied on the +"Info" and "Arguments" tab: + +- :prop_tgt:`XCODE_SCHEME_EXECUTABLE` +- :prop_tgt:`XCODE_SCHEME_ARGUMENTS` +- :prop_tgt:`XCODE_SCHEME_ENVIRONMENT` diff --git a/Help/variable/CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER.rst b/Help/variable/CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER.rst new file mode 100644 index 0000000..37dc0ce --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER +------------------------------------ + +Whether to enable ``Address Sanitizer`` in the Diagnostics +section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_ADDRESS_SANITIZER` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN.rst b/Help/variable/CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN.rst new file mode 100644 index 0000000..05949c3 --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN +----------------------------------------------------- + +Whether to enable ``Detect use of stack after return`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER.rst b/Help/variable/CMAKE_XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER.rst new file mode 100644 index 0000000..81f4974 --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER +---------------------------------------------- + +Whether to disable the ``Main Thread Checker`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS.rst b/Help/variable/CMAKE_XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS.rst new file mode 100644 index 0000000..5e133ac --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS +---------------------------------------- + +Whether to enable ``Dynamic Library Loads`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE.rst b/Help/variable/CMAKE_XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE.rst new file mode 100644 index 0000000..33162d9 --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE +------------------------------------------- + +Whether to enable ``Dynamic Linker API usage`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_GUARD_MALLOC.rst b/Help/variable/CMAKE_XCODE_SCHEME_GUARD_MALLOC.rst new file mode 100644 index 0000000..03d88c2 --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_GUARD_MALLOC.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_GUARD_MALLOC +------------------------------- + +Whether to enable ``Guard Malloc`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_GUARD_MALLOC` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP.rst b/Help/variable/CMAKE_XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP.rst new file mode 100644 index 0000000..fd6135f --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP.rst @@ -0,0 +1,13 @@ +CMAKE_XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP +------------------------------------------- + +Whether to enable the ``Main Thread Checker`` option +``Pause on issues`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_MALLOC_GUARD_EDGES.rst b/Help/variable/CMAKE_XCODE_SCHEME_MALLOC_GUARD_EDGES.rst new file mode 100644 index 0000000..8fedc20 --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_MALLOC_GUARD_EDGES.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_MALLOC_GUARD_EDGES +------------------------------------- + +Whether to enable ``Malloc Guard Edges`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_MALLOC_GUARD_EDGES` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_MALLOC_SCRIBBLE.rst b/Help/variable/CMAKE_XCODE_SCHEME_MALLOC_SCRIBBLE.rst new file mode 100644 index 0000000..cddca7c --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_MALLOC_SCRIBBLE.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_MALLOC_SCRIBBLE +---------------------------------- + +Whether to enable ``Malloc Scribble`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_MALLOC_SCRIBBLE` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_MALLOC_STACK.rst b/Help/variable/CMAKE_XCODE_SCHEME_MALLOC_STACK.rst new file mode 100644 index 0000000..9c83698 --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_MALLOC_STACK.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_MALLOC_STACK +------------------------------- + +Whether to enable ``Malloc Stack`` in the Diagnostics +section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_MALLOC_STACK` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_THREAD_SANITIZER.rst b/Help/variable/CMAKE_XCODE_SCHEME_THREAD_SANITIZER.rst new file mode 100644 index 0000000..c937369 --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_THREAD_SANITIZER.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_THREAD_SANITIZER +----------------------------------- + +Whether to enable ``Thread Sanitizer`` in the Diagnostics +section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_THREAD_SANITIZER` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_THREAD_SANITIZER_STOP.rst b/Help/variable/CMAKE_XCODE_SCHEME_THREAD_SANITIZER_STOP.rst new file mode 100644 index 0000000..eed796c --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_THREAD_SANITIZER_STOP.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_THREAD_SANITIZER_STOP +---------------------------------------- + +Whether to enable ``Thread Sanitizer - Pause on issues`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_THREAD_SANITIZER_STOP` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER.rst b/Help/variable/CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER.rst new file mode 100644 index 0000000..d14ba3f --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER +------------------------------------------------ + +Whether to enable ``Undefined Behavior Sanitizer`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP.rst b/Help/variable/CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP.rst new file mode 100644 index 0000000..f8df304 --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP.rst @@ -0,0 +1,13 @@ +CMAKE_XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP +----------------------------------------------------- + +Whether to enable ``Undefined Behavior Sanitizer`` option +``Pause on issues`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/CMAKE_XCODE_SCHEME_ZOMBIE_OBJECTS.rst b/Help/variable/CMAKE_XCODE_SCHEME_ZOMBIE_OBJECTS.rst new file mode 100644 index 0000000..efc331a --- /dev/null +++ b/Help/variable/CMAKE_XCODE_SCHEME_ZOMBIE_OBJECTS.rst @@ -0,0 +1,12 @@ +CMAKE_XCODE_SCHEME_ZOMBIE_OBJECTS +--------------------------------- + +Whether to enable ``Zombie Objects`` +in the Diagnostics section of the generated Xcode scheme. + +This variable initializes the +:prop_tgt:`XCODE_SCHEME_ZOMBIE_OBJECTS` +property on all targets. + +Please refer to the :variable:`CMAKE_XCODE_GENERATE_SCHEME` variable +documentation to see all Xcode schema related properties. diff --git a/Help/variable/ENV.rst b/Help/variable/ENV.rst index 368152a..98677dd 100644 --- a/Help/variable/ENV.rst +++ b/Help/variable/ENV.rst @@ -1,7 +1,8 @@ ENV --- -Access environment variables. +Read environment variables. -Use the syntax ``$ENV{VAR}`` to read environment variable ``VAR``. See also -the :command:`set` command to set ``ENV{VAR}``. +Use the syntax ``$ENV{VAR}`` to read environment variable ``VAR``. + +See the :command:`set` command to see how to write environment variables. diff --git a/Modules/Compiler/NAG-Fortran.cmake b/Modules/Compiler/NAG-Fortran.cmake index 39aae18..9973feb 100644 --- a/Modules/Compiler/NAG-Fortran.cmake +++ b/Modules/Compiler/NAG-Fortran.cmake @@ -34,3 +34,4 @@ set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-fixed") set(CMAKE_Fortran_FORMAT_FREE_FLAG "-free") set(CMAKE_Fortran_COMPILE_OPTIONS_PIC "-PIC") set(CMAKE_Fortran_COMPILE_OPTIONS_PIE "-PIC") +set(CMAKE_Fortran_RESPONSE_FILE_LINK_FLAG "-Wl,@") diff --git a/Modules/Compiler/XL.cmake b/Modules/Compiler/XL.cmake index a1d7ca1..21fe5e8 100644 --- a/Modules/Compiler/XL.cmake +++ b/Modules/Compiler/XL.cmake @@ -20,6 +20,7 @@ macro(__compiler_xl lang) # Feature flags. set(CMAKE_${lang}_VERBOSE_FLAG "-V") set(CMAKE_${lang}_COMPILE_OPTIONS_PIC "-qpic") + set(CMAKE_${lang}_COMPILE_OPTIONS_PIE "-qpic") set(CMAKE_${lang}_RESPONSE_FILE_FLAG "-qoptfile=") set(CMAKE_${lang}_RESPONSE_FILE_LINK_FLAG "-qoptfile=") diff --git a/Modules/FindCUDA/select_compute_arch.cmake b/Modules/FindCUDA/select_compute_arch.cmake index 22c04df..cf4fc39 100644 --- a/Modules/FindCUDA/select_compute_arch.cmake +++ b/Modules/FindCUDA/select_compute_arch.cmake @@ -18,8 +18,9 @@ # if(CMAKE_CUDA_COMPILER_LOADED) # CUDA as a language - if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA") - set(CUDA_VERSION "${CMAKE_CUDA_COMPILER_VERSION}") + if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA" + AND CMAKE_CUDA_COMPILER_VERSION MATCHES "^([0-9]+\\.[0-9]+)") + set(CUDA_VERSION "${CMAKE_MATCH_1}") endif() endif() diff --git a/Modules/FindMatlab.cmake b/Modules/FindMatlab.cmake index 54e62db..9e13fc3 100644 --- a/Modules/FindMatlab.cmake +++ b/Modules/FindMatlab.cmake @@ -25,6 +25,7 @@ # available on the MCR version, and will yield an error if the MCR is found # instead of the regular Matlab installation. # * ``MEX_COMPILER`` the MEX compiler. +# * ``MCC_COMPILER`` the MCC compiler, included with the Matlab Compiler add-on. # * ``SIMULINK`` the Simulink environment. # # .. note:: @@ -110,7 +111,10 @@ # the whole set of libraries of Matlab # ``Matlab_MEX_COMPILER`` # the mex compiler of Matlab. Currently not used. -# Available only if the component ``MEX_COMPILER`` is asked +# Available only if the component ``MEX_COMPILER`` is requested. +# ``Matlab_MCC_COMPILER`` +# the mcc compiler of Matlab. Included with the Matlab Compiler add-on. +# Available only if the component ``MCC_COMPILER`` is requested. # # Cached variables # """""""""""""""" @@ -1369,7 +1373,7 @@ else() # if the user does not specify the possible installation root, we look for # one installation using the appropriate heuristics. # There is apparently no standard way on Linux. - if(WIN32) + if(CMAKE_HOST_WIN32) _Matlab_find_instances_win32(_matlab_possible_roots_win32) list(APPEND _matlab_possible_roots ${_matlab_possible_roots_win32}) elseif(APPLE) @@ -1420,6 +1424,7 @@ if(DEFINED Matlab_ROOT_DIR_LAST_CACHED) Matlab_INCLUDE_DIRS Matlab_MEX_LIBRARY Matlab_MEX_COMPILER + Matlab_MCC_COMPILER Matlab_MAIN_PROGRAM Matlab_MX_LIBRARY Matlab_ENG_LIBRARY @@ -1660,6 +1665,22 @@ if(_matlab_find_simulink GREATER -1) endif() unset(_matlab_find_simulink) +# component MCC Compiler +list(FIND Matlab_FIND_COMPONENTS MCC_COMPILER _matlab_find_mcc_compiler) +if(_matlab_find_mcc_compiler GREATER -1) + find_program( + Matlab_MCC_COMPILER + "mcc" + PATHS ${Matlab_BINARIES_DIR} + DOC "Matlab MCC compiler" + NO_DEFAULT_PATH + ) + if(Matlab_MCC_COMPILER) + set(Matlab_MCC_COMPILER_FOUND TRUE) + endif() +endif() +unset(_matlab_find_mcc_compiler) + unset(_matlab_lib_dir_for_search) set(Matlab_LIBRARIES ${Matlab_MEX_LIBRARY} ${Matlab_MX_LIBRARY} ${Matlab_ENG_LIBRARY} ${Matlab_MAT_LIBRARY}) diff --git a/Modules/FindXercesC.cmake b/Modules/FindXercesC.cmake index 267c164..55db7ae 100644 --- a/Modules/FindXercesC.cmake +++ b/Modules/FindXercesC.cmake @@ -73,6 +73,10 @@ find_path(XercesC_INCLUDE_DIR DOC "Xerces-C++ include directory") mark_as_advanced(XercesC_INCLUDE_DIR) +if(XercesC_INCLUDE_DIR) + _XercesC_GET_VERSION("${XercesC_INCLUDE_DIR}/xercesc/util/XercesVersion.hpp") +endif() + if(NOT XercesC_LIBRARY) # Find all XercesC libraries find_library(XercesC_LIBRARY_RELEASE @@ -90,10 +94,6 @@ unset(XercesC_VERSION_MAJOR) unset(XercesC_VERSION_MINOR) unset(XercesC_VERSION_PATCH) -if(XercesC_INCLUDE_DIR) - _XercesC_GET_VERSION("${XercesC_INCLUDE_DIR}/xercesc/util/XercesVersion.hpp") -endif() - include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake) FIND_PACKAGE_HANDLE_STANDARD_ARGS(XercesC FOUND_VAR XercesC_FOUND diff --git a/Modules/Platform/Windows-Intel.cmake b/Modules/Platform/Windows-Intel.cmake index ba31001..96b1760 100644 --- a/Modules/Platform/Windows-Intel.cmake +++ b/Modules/Platform/Windows-Intel.cmake @@ -11,7 +11,7 @@ set(__WINDOWS_INTEL 1) include(Platform/Windows-MSVC) macro(__windows_compiler_intel lang) __windows_compiler_msvc(${lang}) - string(REPLACE "<CMAKE_LINKER> /lib" "lib" CMAKE_${lang}_CREATE_STATIC_LIBRARY "${CMAKE_${lang}_CREATE_STATIC_LIBRARY}") + string(REPLACE "<CMAKE_LINKER> /lib" "xilib" CMAKE_${lang}_CREATE_STATIC_LIBRARY "${CMAKE_${lang}_CREATE_STATIC_LIBRARY}") foreach(rule CREATE_SHARED_LIBRARY CREATE_SHARED_MODULE LINK_EXECUTABLE) string(REPLACE "<CMAKE_LINKER>" "xilink" CMAKE_${lang}_${rule} "${CMAKE_${lang}_${rule}}") endforeach() @@ -6,10 +6,12 @@ Introduction CMake is a cross-platform, open-source build system generator. For full documentation visit the `CMake Home Page`_ and the -`CMake Documentation Page`_. +`CMake Documentation Page`_. The `CMake Community Wiki`_ also +references useful guides and recipes. .. _`CMake Home Page`: https://cmake.org .. _`CMake Documentation Page`: https://cmake.org/cmake/help/documentation.html +.. _`CMake Community Wiki`: https://gitlab.kitware.com/cmake/community/wikis/home CMake is maintained and supported by `Kitware`_ and developed in collaboration with a productive community of contributors. diff --git a/Source/CMakeVersion.cmake b/Source/CMakeVersion.cmake index 7363afe..9b8631d 100644 --- a/Source/CMakeVersion.cmake +++ b/Source/CMakeVersion.cmake @@ -1,5 +1,5 @@ # CMake version number components. set(CMake_VERSION_MAJOR 3) set(CMake_VERSION_MINOR 12) -set(CMake_VERSION_PATCH 20180803) +set(CMake_VERSION_PATCH 20180810) #set(CMake_VERSION_RC 1) diff --git a/Source/CPack/IFW/cmCPackIFWGenerator.cxx b/Source/CPack/IFW/cmCPackIFWGenerator.cxx index cf8334a..180c92e 100644 --- a/Source/CPack/IFW/cmCPackIFWGenerator.cxx +++ b/Source/CPack/IFW/cmCPackIFWGenerator.cxx @@ -90,7 +90,7 @@ int cmCPackIFWGenerator::PackageFiles() ifwCmd.c_str(), &output, &output, &retVal, nullptr, this->GeneratorVerbose, cmDuration::zero()); if (!res || retVal) { - cmGeneratedFileStream ofs(ifwTmpFile.c_str()); + cmGeneratedFileStream ofs(ifwTmpFile); ofs << "# Run command: " << ifwCmd << std::endl << "# Output:" << std::endl << output << std::endl; @@ -203,7 +203,7 @@ int cmCPackIFWGenerator::PackageFiles() ifwCmd.c_str(), &output, &output, &retVal, nullptr, this->GeneratorVerbose, cmDuration::zero()); if (!res || retVal) { - cmGeneratedFileStream ofs(ifwTmpFile.c_str()); + cmGeneratedFileStream ofs(ifwTmpFile); ofs << "# Run command: " << ifwCmd << std::endl << "# Output:" << std::endl << output << std::endl; diff --git a/Source/CPack/IFW/cmCPackIFWInstaller.cxx b/Source/CPack/IFW/cmCPackIFWInstaller.cxx index 36cf08c..8f492af 100644 --- a/Source/CPack/IFW/cmCPackIFWInstaller.cxx +++ b/Source/CPack/IFW/cmCPackIFWInstaller.cxx @@ -306,7 +306,7 @@ void cmCPackIFWInstaller::GenerateInstallerFile() } // Output stream - cmGeneratedFileStream fout((this->Directory + "/config/config.xml").data()); + cmGeneratedFileStream fout(this->Directory + "/config/config.xml"); cmXMLWriter xout(fout); xout.StartDocument(); diff --git a/Source/CPack/IFW/cmCPackIFWPackage.cxx b/Source/CPack/IFW/cmCPackIFWPackage.cxx index f24ab69..67e279c 100644 --- a/Source/CPack/IFW/cmCPackIFWPackage.cxx +++ b/Source/CPack/IFW/cmCPackIFWPackage.cxx @@ -528,7 +528,7 @@ void cmCPackIFWPackage::GeneratePackageFile() } // Output stream - cmGeneratedFileStream fout((this->Directory + "/meta/package.xml").data()); + cmGeneratedFileStream fout(this->Directory + "/meta/package.xml"); cmXMLWriter xout(fout); xout.StartDocument(); diff --git a/Source/CPack/IFW/cmCPackIFWRepository.cxx b/Source/CPack/IFW/cmCPackIFWRepository.cxx index a01fc4e..987cad8 100644 --- a/Source/CPack/IFW/cmCPackIFWRepository.cxx +++ b/Source/CPack/IFW/cmCPackIFWRepository.cxx @@ -183,7 +183,7 @@ bool cmCPackIFWRepository::PatchUpdatesXml() this->Directory + "/repository/UpdatesPatch.xml"; // Output stream - cmGeneratedFileStream fout(updatesPatchXml.data()); + cmGeneratedFileStream fout(updatesPatchXml); cmXMLWriter xout(fout); xout.StartDocument(); diff --git a/Source/CPack/cmCPackArchiveGenerator.cxx b/Source/CPack/cmCPackArchiveGenerator.cxx index 593b4bf..ee1070f 100644 --- a/Source/CPack/cmCPackArchiveGenerator.cxx +++ b/Source/CPack/cmCPackArchiveGenerator.cxx @@ -103,7 +103,7 @@ int cmCPackArchiveGenerator::addOneComponentToArchive( */ #define DECLARE_AND_OPEN_ARCHIVE(filename, archive) \ cmGeneratedFileStream gf; \ - gf.Open((filename).c_str(), false, true); \ + gf.Open((filename), false, true); \ if (!GenerateHeader(&gf)) { \ cmCPackLogger(cmCPackLog::LOG_ERROR, \ "Problem to generate Header for archive <" \ diff --git a/Source/CPack/cmCPackDebGenerator.cxx b/Source/CPack/cmCPackDebGenerator.cxx index c53dd32..972fe6f 100644 --- a/Source/CPack/cmCPackDebGenerator.cxx +++ b/Source/CPack/cmCPackDebGenerator.cxx @@ -238,7 +238,7 @@ int cmCPackDebGenerator::createDeb() const std::string strGenWDIR(this->GetOption("GEN_WDIR")); const std::string dbfilename = strGenWDIR + "/debian-binary"; { // the scope is needed for cmGeneratedFileStream - cmGeneratedFileStream out(dbfilename.c_str()); + cmGeneratedFileStream out(dbfilename); out << "2.0"; out << std::endl; // required for valid debian package } @@ -287,7 +287,7 @@ int cmCPackDebGenerator::createDeb() this->GetOption("GEN_CPACK_DEBIAN_PACKAGE_SOURCE"); { // the scope is needed for cmGeneratedFileStream - cmGeneratedFileStream out(ctlfilename.c_str()); + cmGeneratedFileStream out(ctlfilename); out << "Package: " << debian_pkg_name << "\n"; out << "Version: " << debian_pkg_version << "\n"; out << "Section: " << debian_pkg_section << "\n"; @@ -347,7 +347,7 @@ int cmCPackDebGenerator::createDeb() const bool gen_shibs = this->IsOn("CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS") && debian_pkg_shlibs && *debian_pkg_shlibs; if (gen_shibs) { - cmGeneratedFileStream out(shlibsfilename.c_str()); + cmGeneratedFileStream out(shlibsfilename); out << debian_pkg_shlibs; out << std::endl; } @@ -355,7 +355,7 @@ int cmCPackDebGenerator::createDeb() const std::string postinst = strGenWDIR + "/postinst"; const std::string postrm = strGenWDIR + "/postrm"; if (this->IsOn("GEN_CPACK_DEBIAN_GENERATE_POSTINST")) { - cmGeneratedFileStream out(postinst.c_str()); + cmGeneratedFileStream out(postinst); out << "#!/bin/sh\n\n" "set -e\n\n" "if [ \"$1\" = \"configure\" ]; then\n" @@ -363,7 +363,7 @@ int cmCPackDebGenerator::createDeb() "fi\n"; } if (this->IsOn("GEN_CPACK_DEBIAN_GENERATE_POSTRM")) { - cmGeneratedFileStream out(postrm.c_str()); + cmGeneratedFileStream out(postrm); out << "#!/bin/sh\n\n" "set -e\n\n" "if [ \"$1\" = \"remove\" ]; then\n" @@ -412,7 +412,7 @@ int cmCPackDebGenerator::createDeb() // atomic file generation for data.tar { cmGeneratedFileStream fileStream_data_tar; - fileStream_data_tar.Open(filename_data_tar.c_str(), false, true); + fileStream_data_tar.Open(filename_data_tar, false, true); if (!fileStream_data_tar) { cmCPackLogger(cmCPackLog::LOG_ERROR, "Error opening the file \"" @@ -493,7 +493,7 @@ int cmCPackDebGenerator::createDeb() std::string md5filename = strGenWDIR + "/md5sums"; { // the scope is needed for cmGeneratedFileStream - cmGeneratedFileStream out(md5filename.c_str()); + cmGeneratedFileStream out(md5filename); std::string topLevelWithTrailingSlash = this->GetOption("CPACK_TEMPORARY_DIRECTORY"); @@ -528,7 +528,7 @@ int cmCPackDebGenerator::createDeb() // atomic file generation for control.tar { cmGeneratedFileStream fileStream_control_tar; - fileStream_control_tar.Open(filename_control_tar.c_str(), false, true); + fileStream_control_tar.Open(filename_control_tar, false, true); if (!fileStream_control_tar) { cmCPackLogger(cmCPackLog::LOG_ERROR, "Error opening the file \"" << filename_control_tar @@ -662,7 +662,7 @@ int cmCPackDebGenerator::createDeb() std::string const outputPath = outputDir + "/" + outputName; std::string const tlDir = strGenWDIR + "/"; cmGeneratedFileStream debStream; - debStream.Open(outputPath.c_str(), false, true); + debStream.Open(outputPath, false, true); cmArchiveWrite deb(debStream, cmArchiveWrite::CompressNone, "arbsd"); // uid/gid should be the one of the root user, and this root user has diff --git a/Source/CPack/cmCPackGenerator.cxx b/Source/CPack/cmCPackGenerator.cxx index 7014676..6347eed 100644 --- a/Source/CPack/cmCPackGenerator.cxx +++ b/Source/CPack/cmCPackGenerator.cxx @@ -288,7 +288,7 @@ int cmCPackGenerator::InstallProjectViaInstallCommands( if (!resB || retVal) { std::string tmpFile = this->GetOption("CPACK_TOPLEVEL_DIRECTORY"); tmpFile += "/InstallOutput.log"; - cmGeneratedFileStream ofs(tmpFile.c_str()); + cmGeneratedFileStream ofs(tmpFile); ofs << "# Run command: " << ic << std::endl << "# Output:" << std::endl << output << std::endl; @@ -369,7 +369,7 @@ int cmCPackGenerator::InstallProjectViaInstalledDirectories( inFile += '/'; } for (cmsys::RegularExpression& reg : ignoreFilesRegex) { - if (reg.find(inFile.c_str())) { + if (reg.find(inFile)) { cmCPackLogger(cmCPackLog::LOG_VERBOSE, "Ignore file: " << inFile << std::endl); skip = true; @@ -660,7 +660,7 @@ int cmCPackGenerator::RunPreinstallTarget( if (!resB || retVal) { std::string tmpFile = this->GetOption("CPACK_TOPLEVEL_DIRECTORY"); tmpFile += "/PreinstallOutput.log"; - cmGeneratedFileStream ofs(tmpFile.c_str()); + cmGeneratedFileStream ofs(tmpFile); ofs << "# Run command: " << buildCommand << std::endl << "# Directory: " << installDirectory << std::endl << "# Output:" << std::endl diff --git a/Source/CPack/cmCPackNSISGenerator.cxx b/Source/CPack/cmCPackNSISGenerator.cxx index e46b2cf..f75a750 100644 --- a/Source/CPack/cmCPackNSISGenerator.cxx +++ b/Source/CPack/cmCPackNSISGenerator.cxx @@ -305,7 +305,7 @@ int cmCPackNSISGenerator::PackageFiles() nsisCmd.c_str(), &output, &output, &retVal, nullptr, this->GeneratorVerbose, cmDuration::zero()); if (!res || retVal) { - cmGeneratedFileStream ofs(tmpFile.c_str()); + cmGeneratedFileStream ofs(tmpFile); ofs << "# Run command: " << nsisCmd << std::endl << "# Output:" << std::endl << output << std::endl; @@ -416,7 +416,7 @@ int cmCPackNSISGenerator::InitializeInternal() const char* topDir = this->GetOption("CPACK_TOPLEVEL_DIRECTORY"); std::string tmpFile = topDir ? topDir : "."; tmpFile += "/NSISOutput.log"; - cmGeneratedFileStream ofs(tmpFile.c_str()); + cmGeneratedFileStream ofs(tmpFile); ofs << "# Run command: " << nsisCmd << std::endl << "# Output:" << std::endl << output << std::endl; @@ -726,7 +726,7 @@ std::string cmCPackNSISGenerator::CreateComponentDescription( cmSystemTools::IsOn(this->GetOption("CPACK_ZIP_NEED_QUOTES")); unsigned long totalSize = 0; { // the scope is needed for cmGeneratedFileStream - cmGeneratedFileStream out(zipListFileName.c_str()); + cmGeneratedFileStream out(zipListFileName); for (std::string const& file : component->Files) { if (needQuotesInFile) { out << "\""; @@ -754,7 +754,7 @@ std::string cmCPackNSISGenerator::CreateComponentDescription( if (!res || retVal) { std::string tmpFile = this->GetOption("CPACK_TOPLEVEL_DIRECTORY"); tmpFile += "/CompressZip.log"; - cmGeneratedFileStream ofs(tmpFile.c_str()); + cmGeneratedFileStream ofs(tmpFile); ofs << "# Run command: " << cmd << std::endl << "# Output:" << std::endl << output << std::endl; diff --git a/Source/CPack/cmCPackProductBuildGenerator.cxx b/Source/CPack/cmCPackProductBuildGenerator.cxx index 4ca0fa8..76b3275 100644 --- a/Source/CPack/cmCPackProductBuildGenerator.cxx +++ b/Source/CPack/cmCPackProductBuildGenerator.cxx @@ -144,10 +144,10 @@ bool cmCPackProductBuildGenerator::RunProductBuild(const std::string& command) tmpFile += "/ProductBuildOutput.log"; cmCPackLogger(cmCPackLog::LOG_VERBOSE, "Execute: " << command << std::endl); - std::string output, error_output; + std::string output; int retVal = 1; bool res = cmSystemTools::RunSingleCommand( - command.c_str(), &output, &error_output, &retVal, nullptr, + command.c_str(), &output, &output, &retVal, nullptr, this->GeneratorVerbose, cmDuration::zero()); cmCPackLogger(cmCPackLog::LOG_VERBOSE, "Done running command" << std::endl); if (!res || retVal) { diff --git a/Source/CTest/cmCTestBuildHandler.cxx b/Source/CTest/cmCTestBuildHandler.cxx index 8527d54..3f6654b 100644 --- a/Source/CTest/cmCTestBuildHandler.cxx +++ b/Source/CTest/cmCTestBuildHandler.cxx @@ -589,7 +589,7 @@ void cmCTestBuildHandler::GenerateXMLLogScraped(cmXMLWriter& xml) for (cmCTestCompileErrorWarningRex& rit : this->ErrorWarningFileLineRegex) { cmsys::RegularExpression* re = &rit.RegularExpression; - if (re->find(cm->Text.c_str())) { + if (re->find(cm->Text)) { cm->SourceFile = re->match(rit.FileIndex); // At this point we need to make this->SourceFile relative to // the source root of the project, so cvs links will work @@ -742,7 +742,7 @@ void cmCTestBuildHandler::LaunchHelper::WriteLauncherConfig() // Give some testing configuration information to the launcher. std::string fname = this->Handler->CTestLaunchDir; fname += "/CTestLaunchConfig.cmake"; - cmGeneratedFileStream fout(fname.c_str()); + cmGeneratedFileStream fout(fname); std::string srcdir = this->CTest->GetCTestConfiguration("SourceDirectory"); fout << "set(CTEST_SOURCE_DIRECTORY \"" << srcdir << "\")\n"; } @@ -757,7 +757,7 @@ void cmCTestBuildHandler::LaunchHelper::WriteScrapeMatchers( fname += "/Custom"; fname += purpose; fname += ".txt"; - cmGeneratedFileStream fout(fname.c_str()); + cmGeneratedFileStream fout(fname); for (std::string const& m : matchers) { fout << m << "\n"; } diff --git a/Source/CTest/cmCTestCoverageHandler.cxx b/Source/CTest/cmCTestCoverageHandler.cxx index 43a2bae..1d9a5f7 100644 --- a/Source/CTest/cmCTestCoverageHandler.cxx +++ b/Source/CTest/cmCTestCoverageHandler.cxx @@ -1075,7 +1075,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( if (line.empty()) { // Ignore empty line; probably style 2 - } else if (st1re1.find(line.c_str())) { + } else if (st1re1.find(line)) { if (gcovStyle == 0) { gcovStyle = 1; } @@ -1088,7 +1088,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( actualSourceFile.clear(); sourceFile = st1re1.match(2); - } else if (st1re2.find(line.c_str())) { + } else if (st1re2.find(line)) { if (gcovStyle == 0) { gcovStyle = 1; } @@ -1100,7 +1100,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( } gcovFile = st1re2.match(1); - } else if (st2re1.find(line.c_str())) { + } else if (st2re1.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } @@ -1113,7 +1113,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( actualSourceFile.clear(); sourceFile = st2re1.match(1); - } else if (st2re2.find(line.c_str())) { + } else if (st2re2.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } @@ -1123,7 +1123,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( cont->Error++; break; } - } else if (st2re3.find(line.c_str())) { + } else if (st2re3.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } @@ -1135,7 +1135,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( } gcovFile = st2re3.match(2); - } else if (st2re4.find(line.c_str())) { + } else if (st2re4.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } @@ -1150,7 +1150,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( "Warning: " << st2re4.match(1) << " had unexpected EOF" << std::endl, this->Quiet); - } else if (st2re5.find(line.c_str())) { + } else if (st2re5.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } @@ -1165,7 +1165,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( "Warning: Cannot open file: " << st2re5.match(1) << std::endl, this->Quiet); - } else if (st2re6.find(line.c_str())) { + } else if (st2re6.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } diff --git a/Source/CTest/cmCTestLaunch.cxx b/Source/CTest/cmCTestLaunch.cxx index 0bffde3..6cd1c09 100644 --- a/Source/CTest/cmCTestLaunch.cxx +++ b/Source/CTest/cmCTestLaunch.cxx @@ -345,7 +345,7 @@ void cmCTestLaunch::WriteXML() logXML += ".xml"; // Use cmGeneratedFileStream to atomically create the report file. - cmGeneratedFileStream fxml(logXML.c_str()); + cmGeneratedFileStream fxml(logXML); cmXMLWriter xml(fxml, 2); cmXMLElement e2(xml, "Failure"); e2.Attribute("type", this->IsError() ? "Error" : "Warning"); @@ -584,7 +584,7 @@ bool cmCTestLaunch::Match(std::string const& line, std::vector<cmsys::RegularExpression>& regexps) { for (cmsys::RegularExpression& r : regexps) { - if (r.find(line.c_str())) { + if (r.find(line)) { return true; } } diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 927797a..ef0a49d 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -49,7 +49,7 @@ void cmCTestRunTest::CheckOutput(std::string const& line) // Check for TIMEOUT_AFTER_MATCH property. if (!this->TestProperties->TimeoutRegularExpressions.empty()) { for (auto& reg : this->TestProperties->TimeoutRegularExpressions) { - if (reg.first.find(this->ProcessOutput.c_str())) { + if (reg.first.find(this->ProcessOutput)) { cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->GetIndex() << ": " @@ -148,7 +148,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) this->FailedDependencies.empty()) { bool found = false; for (auto& pass : this->TestProperties->RequiredRegularExpressions) { - if (pass.first.find(this->ProcessOutput.c_str())) { + if (pass.first.find(this->ProcessOutput)) { found = true; reason = "Required regular expression found."; break; @@ -168,7 +168,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) if (!this->TestProperties->ErrorRegularExpressions.empty() && this->FailedDependencies.empty()) { for (auto& pass : this->TestProperties->ErrorRegularExpressions) { - if (pass.first.find(this->ProcessOutput.c_str())) { + if (pass.first.find(this->ProcessOutput)) { reason = "Error regular expression found in output."; reason += " Regex=["; reason += pass.second; @@ -616,10 +616,10 @@ void cmCTestRunTest::DartProcessing() { if (!this->ProcessOutput.empty() && this->ProcessOutput.find("<DartMeasurement") != std::string::npos) { - if (this->TestHandler->DartStuff.find(this->ProcessOutput.c_str())) { + if (this->TestHandler->DartStuff.find(this->ProcessOutput)) { this->TestResult.DartString = this->TestHandler->DartStuff.match(1); // keep searching and replacing until none are left - while (this->TestHandler->DartStuff1.find(this->ProcessOutput.c_str())) { + while (this->TestHandler->DartStuff1.find(this->ProcessOutput)) { // replace the exact match for the string cmSystemTools::ReplaceString( this->ProcessOutput, this->TestHandler->DartStuff1.match(1).c_str(), diff --git a/Source/CTest/cmCTestScriptHandler.cxx b/Source/CTest/cmCTestScriptHandler.cxx index 333c899..6b62bb4 100644 --- a/Source/CTest/cmCTestScriptHandler.cxx +++ b/Source/CTest/cmCTestScriptHandler.cxx @@ -792,7 +792,7 @@ int cmCTestScriptHandler::RunConfigurationDashboard() cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Write CMake output to file: " << cmakeOutputFile << std::endl); - cmGeneratedFileStream fout(cmakeOutputFile.c_str()); + cmGeneratedFileStream fout(cmakeOutputFile); if (fout) { fout << output.c_str(); } else { @@ -856,7 +856,7 @@ bool cmCTestScriptHandler::WriteInitialCache(const char* directory, { std::string cacheFile = directory; cacheFile += "/CMakeCache.txt"; - cmGeneratedFileStream fout(cacheFile.c_str()); + cmGeneratedFileStream fout(cacheFile); if (!fout) { return false; } diff --git a/Source/CTest/cmCTestStartCommand.cxx b/Source/CTest/cmCTestStartCommand.cxx index 3911540..47006c1 100644 --- a/Source/CTest/cmCTestStartCommand.cxx +++ b/Source/CTest/cmCTestStartCommand.cxx @@ -121,7 +121,7 @@ bool cmCTestStartCommand::InitialPass(std::vector<std::string> const& args, // Log startup actions. std::string startLogFile = binaryDir + "/Testing/Temporary/LastStart.log"; - cmGeneratedFileStream ofs(startLogFile.c_str()); + cmGeneratedFileStream ofs(startLogFile); if (!ofs) { cmCTestLog(this->CTest, ERROR_MESSAGE, "Cannot create log file: LastStart.log" << std::endl); diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 91b92a3..d1d8d08 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -2310,7 +2310,7 @@ bool cmCTestTestHandler::AddTest(const std::vector<std::string>& args) this->Quiet); if (this->UseExcludeRegExpFlag && this->UseExcludeRegExpFirst && - this->ExcludeTestsRegularExpression.find(testname.c_str())) { + this->ExcludeTestsRegularExpression.find(testname)) { return true; } if (this->MemCheck) { @@ -2365,10 +2365,10 @@ bool cmCTestTestHandler::AddTest(const std::vector<std::string>& args) test.SkipReturnCode = -1; test.PreviousRuns = 0; if (this->UseIncludeRegExpFlag && - !this->IncludeTestsRegularExpression.find(testname.c_str())) { + !this->IncludeTestsRegularExpression.find(testname)) { test.IsInBasedOnREOptions = false; } else if (this->UseExcludeRegExpFlag && !this->UseExcludeRegExpFirst && - this->ExcludeTestsRegularExpression.find(testname.c_str())) { + this->ExcludeTestsRegularExpression.find(testname)) { test.IsInBasedOnREOptions = false; } this->TestList.push_back(test); diff --git a/Source/Checks/cm_cxx14_check.cmake b/Source/Checks/cm_cxx14_check.cmake index a78ba35..38606b9 100644 --- a/Source/Checks/cm_cxx14_check.cmake +++ b/Source/Checks/cm_cxx14_check.cmake @@ -1,5 +1,5 @@ set(CMake_CXX14_BROKEN 0) -if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|PGI") if(NOT CMAKE_CXX14_STANDARD_COMPILE_OPTION) set(CMake_CXX14_WORKS 0) endif() diff --git a/Source/Checks/cm_cxx14_check.cpp b/Source/Checks/cm_cxx14_check.cpp index f5806a9..9369ba2 100644 --- a/Source/Checks/cm_cxx14_check.cpp +++ b/Source/Checks/cm_cxx14_check.cpp @@ -1,5 +1,8 @@ #include <cstdio> +#include <memory> + int main() { - return 0; + std::unique_ptr<int> u(new int(0)); + return *u; } diff --git a/Source/Checks/cm_cxx17_check.cmake b/Source/Checks/cm_cxx17_check.cmake index 83d3971..4da2fd7 100644 --- a/Source/Checks/cm_cxx17_check.cmake +++ b/Source/Checks/cm_cxx17_check.cmake @@ -1,5 +1,5 @@ set(CMake_CXX17_BROKEN 0) -if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|PGI") if(NOT CMAKE_CXX17_STANDARD_COMPILE_OPTION) set(CMake_CXX17_WORKS 0) endif() diff --git a/Source/Checks/cm_cxx17_check.cpp b/Source/Checks/cm_cxx17_check.cpp index 2cbf1d5..4e89184 100644 --- a/Source/Checks/cm_cxx17_check.cpp +++ b/Source/Checks/cm_cxx17_check.cpp @@ -1,7 +1,9 @@ #include <cstdio> +#include <memory> #include <unordered_map> int main() { - return 0; + std::unique_ptr<int> u(new int(0)); + return *u; } diff --git a/Source/cmBuildNameCommand.cxx b/Source/cmBuildNameCommand.cxx index 5f54338..ddff686 100644 --- a/Source/cmBuildNameCommand.cxx +++ b/Source/cmBuildNameCommand.cxx @@ -40,8 +40,8 @@ bool cmBuildNameCommand::InitialPass(std::vector<std::string> const& args, cmSystemTools::RunSingleCommand("uname -a", &buildname, &buildname); if (!buildname.empty()) { std::string RegExp = "([^ ]*) [^ ]* ([^ ]*) "; - cmsys::RegularExpression reg(RegExp.c_str()); - if (reg.find(buildname.c_str())) { + cmsys::RegularExpression reg(RegExp); + if (reg.find(buildname)) { buildname = reg.match(1) + "-" + reg.match(2); } } diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 95edd8b..3d3d526 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -818,7 +818,7 @@ bool cmCTest::OpenOutputFile(const std::string& path, const std::string& name, } } std::string filename = testingDir + "/" + name; - stream.Open(filename.c_str()); + stream.Open(filename); if (!stream) { cmCTestLog(this, ERROR_MESSAGE, "Problem opening file: " << filename << std::endl); diff --git a/Source/cmCacheManager.cxx b/Source/cmCacheManager.cxx index c6a1cff..5091c97 100644 --- a/Source/cmCacheManager.cxx +++ b/Source/cmCacheManager.cxx @@ -234,7 +234,7 @@ bool cmCacheManager::SaveCache(const std::string& path, cmMessenger* messenger) { std::string cacheFile = path; cacheFile += "/CMakeCache.txt"; - cmGeneratedFileStream fout(cacheFile.c_str()); + cmGeneratedFileStream fout(cacheFile); fout.SetCopyIfDifferent(true); if (!fout) { cmSystemTools::Error("Unable to open cache file for save. ", diff --git a/Source/cmDependsC.cxx b/Source/cmDependsC.cxx index 34c0e94..f7dd33b 100644 --- a/Source/cmDependsC.cxx +++ b/Source/cmDependsC.cxx @@ -174,7 +174,7 @@ bool cmDependsC::WriteDependencies(const std::set<std::string>& sources, // Complain if the file cannot be found and matches the complain // regex. if (fullName.empty() && - this->IncludeRegexComplain.find(current.FileName.c_str())) { + this->IncludeRegexComplain.find(current.FileName)) { cmSystemTools::Error("Cannot find file \"", current.FileName.c_str(), "\"."); return false; @@ -359,7 +359,7 @@ void cmDependsC::Scan(std::istream& is, const char* directory, } // Match include directives. - if (this->IncludeRegexLine.find(line.c_str())) { + if (this->IncludeRegexLine.find(line)) { // Get the file being included. UnscannedEntry entry; entry.FileName = this->IncludeRegexLine.match(2); @@ -383,7 +383,7 @@ void cmDependsC::Scan(std::istream& is, const char* directory, // file their own directory by simply using "filename.h" (#12619) // This kind of problem will be fixed when a more // preprocessor-like implementation of this scanner is created. - if (this->IncludeRegexScan.find(entry.FileName.c_str())) { + if (this->IncludeRegexScan.find(entry.FileName)) { newCacheEntry->UnscannedEntries.push_back(entry); if (this->Encountered.find(entry.FileName) == this->Encountered.end()) { @@ -449,7 +449,7 @@ void cmDependsC::ParseTransform(std::string const& xform) void cmDependsC::TransformLine(std::string& line) { // Check for a transform rule match. Return if none. - if (!this->IncludeRegexTransform.find(line.c_str())) { + if (!this->IncludeRegexTransform.find(line)) { return; } TransformRulesType::const_iterator tri = diff --git a/Source/cmDependsFortran.cxx b/Source/cmDependsFortran.cxx index dbea15a..a04cee7 100644 --- a/Source/cmDependsFortran.cxx +++ b/Source/cmDependsFortran.cxx @@ -178,7 +178,7 @@ bool cmDependsFortran::Finalize(std::ostream& makeDepends, // Store the list of modules provided by this target. std::string fiName = this->TargetDirectory; fiName += "/fortran.internal"; - cmGeneratedFileStream fiStream(fiName.c_str()); + cmGeneratedFileStream fiStream(fiName); fiStream << "# The fortran modules provided by this target.\n"; fiStream << "provides\n"; std::set<std::string> const& provides = this->Internal->TargetProvides; @@ -190,7 +190,7 @@ bool cmDependsFortran::Finalize(std::ostream& makeDepends, if (!provides.empty()) { std::string fcName = this->TargetDirectory; fcName += "/cmake_clean_Fortran.cmake"; - cmGeneratedFileStream fcStream(fcName.c_str()); + cmGeneratedFileStream fcStream(fcName); fcStream << "# Remove fortran modules provided by this target.\n"; fcStream << "FILE(REMOVE"; std::string currentBinDir = diff --git a/Source/cmExportCommand.cxx b/Source/cmExportCommand.cxx index 4e1d771..c6ec5bf 100644 --- a/Source/cmExportCommand.cxx +++ b/Source/cmExportCommand.cxx @@ -237,7 +237,7 @@ bool cmExportCommand::HandlePackage(std::vector<std::string> const& args) } const char* packageExpr = "^[A-Za-z0-9_.-]+$"; cmsys::RegularExpression packageRegex(packageExpr); - if (!packageRegex.find(package.c_str())) { + if (!packageRegex.find(package)) { std::ostringstream e; e << "PACKAGE given invalid package name \"" << package << "\". " << "Package names must match \"" << packageExpr << "\"."; @@ -339,7 +339,7 @@ void cmExportCommand::StorePackageRegistryDir(std::string const& package, fname += "/"; fname += hash; if (!cmSystemTools::FileExists(fname)) { - cmGeneratedFileStream entry(fname.c_str(), true); + cmGeneratedFileStream entry(fname, true); if (entry) { entry << content << "\n"; } else { diff --git a/Source/cmExportFileGenerator.cxx b/Source/cmExportFileGenerator.cxx index ec68fce..002fae6 100644 --- a/Source/cmExportFileGenerator.cxx +++ b/Source/cmExportFileGenerator.cxx @@ -75,7 +75,7 @@ bool cmExportFileGenerator::GenerateImportFile() } else { // Generate atomically and with copy-if-different. std::unique_ptr<cmGeneratedFileStream> ap( - new cmGeneratedFileStream(this->MainImportFile.c_str(), true)); + new cmGeneratedFileStream(this->MainImportFile, true)); ap->SetCopyIfDifferent(true); foutPtr = std::move(ap); } diff --git a/Source/cmExportInstallFileGenerator.cxx b/Source/cmExportInstallFileGenerator.cxx index 3595708..bfb7a05 100644 --- a/Source/cmExportInstallFileGenerator.cxx +++ b/Source/cmExportInstallFileGenerator.cxx @@ -288,7 +288,7 @@ bool cmExportInstallFileGenerator::GenerateImportFileConfig( fileName += this->FileExt; // Open the output file to generate it. - cmGeneratedFileStream exportFileStream(fileName.c_str(), true); + cmGeneratedFileStream exportFileStream(fileName, true); if (!exportFileStream) { std::string se = cmSystemTools::GetLastSystemError(); std::ostringstream e; diff --git a/Source/cmExportLibraryDependenciesCommand.cxx b/Source/cmExportLibraryDependenciesCommand.cxx index 50f8cb0..b4b2962 100644 --- a/Source/cmExportLibraryDependenciesCommand.cxx +++ b/Source/cmExportLibraryDependenciesCommand.cxx @@ -54,7 +54,7 @@ void cmExportLibraryDependenciesCommand::ConstFinalPass() const cm::make_unique<cmsys::ofstream>(this->Filename.c_str(), std::ios::app); } else { std::unique_ptr<cmGeneratedFileStream> ap( - new cmGeneratedFileStream(this->Filename.c_str(), true)); + new cmGeneratedFileStream(this->Filename, true)); ap->SetCopyIfDifferent(true); foutPtr = std::move(ap); } diff --git a/Source/cmExtraCodeBlocksGenerator.cxx b/Source/cmExtraCodeBlocksGenerator.cxx index b636c73..fbf6560 100644 --- a/Source/cmExtraCodeBlocksGenerator.cxx +++ b/Source/cmExtraCodeBlocksGenerator.cxx @@ -199,7 +199,7 @@ void cmExtraCodeBlocksGenerator::CreateNewProjectFile( const std::vector<cmLocalGenerator*>& lgs, const std::string& filename) { const cmMakefile* mf = lgs[0]->GetMakefile(); - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } @@ -476,7 +476,7 @@ std::string cmExtraCodeBlocksGenerator::CreateDummyTargetFile( filename += "/"; filename += target->GetName(); filename += ".objlib"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (fout) { /* clang-format off */ fout << "# This is a dummy file for the OBJECT library " diff --git a/Source/cmExtraCodeLiteGenerator.cxx b/Source/cmExtraCodeLiteGenerator.cxx index b76efb2..582a2e7 100644 --- a/Source/cmExtraCodeLiteGenerator.cxx +++ b/Source/cmExtraCodeLiteGenerator.cxx @@ -77,7 +77,7 @@ void cmExtraCodeLiteGenerator::Generate() } } - cmGeneratedFileStream fout(workspaceFileName.c_str()); + cmGeneratedFileStream fout(workspaceFileName); cmXMLWriter xml(fout); xml.StartDocument("utf-8"); @@ -249,7 +249,7 @@ void cmExtraCodeLiteGenerator::CreateNewProjectFile( const std::vector<cmLocalGenerator*>& lgs, const std::string& filename) { const cmMakefile* mf = lgs[0]->GetMakefile(); - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } @@ -547,7 +547,7 @@ void cmExtraCodeLiteGenerator::CreateNewProjectFile( const cmGeneratorTarget* gt, const std::string& filename) { const cmMakefile* mf = gt->Makefile; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } diff --git a/Source/cmExtraEclipseCDT4Generator.cxx b/Source/cmExtraEclipseCDT4Generator.cxx index 14448f5..313d46b 100644 --- a/Source/cmExtraEclipseCDT4Generator.cxx +++ b/Source/cmExtraEclipseCDT4Generator.cxx @@ -98,7 +98,7 @@ void cmExtraEclipseCDT4Generator::Generate() std::string eclipseVersion = mf->GetSafeDefinition("CMAKE_ECLIPSE_VERSION"); cmsys::RegularExpression regex(".*([0-9]+\\.[0-9]+).*"); - if (regex.find(eclipseVersion.c_str())) { + if (regex.find(eclipseVersion)) { unsigned int majorVersion = 0; unsigned int minorVersion = 0; int res = @@ -176,7 +176,7 @@ void cmExtraEclipseCDT4Generator::CreateSourceProjectFile() this->GetPathBasename(this->HomeDirectory)); const std::string filename = this->HomeDirectory + "/.project"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } @@ -261,7 +261,7 @@ void cmExtraEclipseCDT4Generator::CreateProjectFile() const std::string filename = this->HomeOutputDirectory + "/.project"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } @@ -555,7 +555,7 @@ void cmExtraEclipseCDT4Generator::AppendIncludeDirectories( // Frameworks/ part has to be stripped // /System/Library/Frameworks/GLUT.framework/Headers cmsys::RegularExpression frameworkRx("(.+/Frameworks)/.+\\.framework/"); - if (frameworkRx.find(dir.c_str())) { + if (frameworkRx.find(dir)) { dir = frameworkRx.match(1); } @@ -582,7 +582,7 @@ void cmExtraEclipseCDT4Generator::CreateCProjectFile() const const std::string filename = this->HomeOutputDirectory + "/.cproject"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } diff --git a/Source/cmExtraKateGenerator.cxx b/Source/cmExtraKateGenerator.cxx index df18715..f5c4c93 100644 --- a/Source/cmExtraKateGenerator.cxx +++ b/Source/cmExtraKateGenerator.cxx @@ -58,7 +58,7 @@ void cmExtraKateGenerator::CreateKateProjectFile( { std::string filename = lg->GetBinaryDirectory(); filename += "/.kateproject"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } @@ -215,7 +215,7 @@ void cmExtraKateGenerator::CreateDummyKateProjectFile( filename += "/"; filename += this->ProjectName; filename += ".kateproject"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } diff --git a/Source/cmExtraSublimeTextGenerator.cxx b/Source/cmExtraSublimeTextGenerator.cxx index fb85a68..14e7927 100644 --- a/Source/cmExtraSublimeTextGenerator.cxx +++ b/Source/cmExtraSublimeTextGenerator.cxx @@ -91,7 +91,7 @@ void cmExtraSublimeTextGenerator::CreateNewProjectFile( { const cmMakefile* mf = lgs[0]->GetMakefile(); - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } diff --git a/Source/cmFileCommand.cxx b/Source/cmFileCommand.cxx index 4c288f5..0c80319 100644 --- a/Source/cmFileCommand.cxx +++ b/Source/cmFileCommand.cxx @@ -208,16 +208,20 @@ bool cmFileCommand::HandleWriteCommand(std::vector<std::string> const& args, cmSystemTools::MakeDirectory(dir); mode_t mode = 0; + bool writable = false; // Set permissions to writable if (cmSystemTools::GetPermissions(fileName.c_str(), mode)) { - cmSystemTools::SetPermissions(fileName.c_str(), #if defined(_MSC_VER) || defined(__MINGW32__) - mode | S_IWRITE + writable = mode & S_IWRITE; + mode_t newMode = mode | S_IWRITE; #else - mode | S_IWUSR | S_IWGRP + writable = mode & S_IWUSR; + mode_t newMode = mode | S_IWUSR | S_IWGRP; #endif - ); + if (!writable) { + cmSystemTools::SetPermissions(fileName.c_str(), newMode); + } } // If GetPermissions fails, pretend like it is ok. File open will fail if // the file is not writable @@ -242,7 +246,7 @@ bool cmFileCommand::HandleWriteCommand(std::vector<std::string> const& args, return false; } file.close(); - if (mode) { + if (mode && !writable) { cmSystemTools::SetPermissions(fileName.c_str(), mode); } return true; @@ -659,7 +663,7 @@ bool cmFileCommand::HandleStringsCommand(std::vector<std::string> const& args) // The current line has been terminated. Check if the current // string matches the requirements. The length may now be as // low as zero since blank lines are allowed. - if (s.length() >= minlen && (!have_regex || regex.find(s.c_str()))) { + if (s.length() >= minlen && (!have_regex || regex.find(s))) { output_size += static_cast<int>(s.size()) + 1; if (limit_output >= 0 && output_size >= limit_output) { s.clear(); @@ -675,7 +679,7 @@ bool cmFileCommand::HandleStringsCommand(std::vector<std::string> const& args) // string matches the requirements. We require that the length // be at least one no matter what the user specified. if (s.length() >= minlen && !s.empty() && - (!have_regex || regex.find(s.c_str()))) { + (!have_regex || regex.find(s))) { output_size += static_cast<int>(s.size()) + 1; if (limit_output >= 0 && output_size >= limit_output) { s.clear(); @@ -692,7 +696,7 @@ bool cmFileCommand::HandleStringsCommand(std::vector<std::string> const& args) if (maxlen > 0 && s.size() == maxlen) { // Terminate a string if the maximum length is reached. - if (s.length() >= minlen && (!have_regex || regex.find(s.c_str()))) { + if (s.length() >= minlen && (!have_regex || regex.find(s))) { output_size += static_cast<int>(s.size()) + 1; if (limit_output >= 0 && output_size >= limit_output) { s.clear(); @@ -708,7 +712,7 @@ bool cmFileCommand::HandleStringsCommand(std::vector<std::string> const& args) // input file or the input size limit. Check if the current string // matches the requirements. if ((!limit_count || strings.size() < limit_count) && !s.empty() && - s.length() >= minlen && (!have_regex || regex.find(s.c_str()))) { + s.length() >= minlen && (!have_regex || regex.find(s))) { output_size += static_cast<int>(s.size()) + 1; if (limit_output < 0 || output_size < limit_output) { strings.push_back(s); diff --git a/Source/cmFindPackageCommand.cxx b/Source/cmFindPackageCommand.cxx index 7ac0cdf..598b9b6 100644 --- a/Source/cmFindPackageCommand.cxx +++ b/Source/cmFindPackageCommand.cxx @@ -337,7 +337,7 @@ bool cmFindPackageCommand::InitialPass(std::vector<std::string> const& args, return false; } this->Configs.push_back(args[i]); - } else if (!haveVersion && version.find(args[i].c_str())) { + } else if (!haveVersion && version.find(args[i])) { haveVersion = true; this->Version = args[i]; } else { diff --git a/Source/cmFindPackageCommand.h b/Source/cmFindPackageCommand.h index d8c7ab3..48f17ef 100644 --- a/Source/cmFindPackageCommand.h +++ b/Source/cmFindPackageCommand.h @@ -7,6 +7,7 @@ #include "cm_kwiml.h" #include <cstddef> +#include <functional> #include <map> #include <set> #include <string> @@ -15,14 +16,12 @@ // IWYU insists we should forward-declare instead of including <functional>, // but we cannot forward-declare reliably because some C++ standard libraries // put the template in an inline namespace. -#ifdef CMAKE_IWYU +#ifdef CMAKE_IWYU_FORWARD_STD_HASH /* clang-format off */ namespace std { template <class T> struct hash; } /* clang-format on */ -#else -# include <functional> #endif #include "cmFindCommon.h" diff --git a/Source/cmGeneratedFileStream.cxx b/Source/cmGeneratedFileStream.cxx index 6bdf3a4..186ec8c 100644 --- a/Source/cmGeneratedFileStream.cxx +++ b/Source/cmGeneratedFileStream.cxx @@ -24,8 +24,8 @@ cmGeneratedFileStream::cmGeneratedFileStream(Encoding encoding) #endif } -cmGeneratedFileStream::cmGeneratedFileStream(const char* name, bool quiet, - Encoding encoding) +cmGeneratedFileStream::cmGeneratedFileStream(std::string const& name, + bool quiet, Encoding encoding) : cmGeneratedFileStreamBase(name) , Stream(TempName.c_str()) { @@ -54,7 +54,7 @@ cmGeneratedFileStream::~cmGeneratedFileStream() this->Okay = !this->fail(); } -cmGeneratedFileStream& cmGeneratedFileStream::Open(const char* name, +cmGeneratedFileStream& cmGeneratedFileStream::Open(std::string const& name, bool quiet, bool binaryFlag) { // Store the file name and construct the temporary file name. @@ -114,7 +114,7 @@ cmGeneratedFileStreamBase::cmGeneratedFileStreamBase() { } -cmGeneratedFileStreamBase::cmGeneratedFileStreamBase(const char* name) +cmGeneratedFileStreamBase::cmGeneratedFileStreamBase(std::string const& name) : Name() , TempName() , CopyIfDifferent(false) @@ -130,7 +130,7 @@ cmGeneratedFileStreamBase::~cmGeneratedFileStreamBase() this->Close(); } -void cmGeneratedFileStreamBase::Open(const char* name) +void cmGeneratedFileStreamBase::Open(std::string const& name) { // Save the original name of the file. this->Name = name; @@ -168,12 +168,12 @@ bool cmGeneratedFileStreamBase::Close() // destination atomically. if (this->Compress) { std::string gzname = this->TempName + ".temp.gz"; - if (this->CompressFile(this->TempName.c_str(), gzname.c_str())) { - this->RenameFile(gzname.c_str(), resname.c_str()); + if (this->CompressFile(this->TempName, gzname)) { + this->RenameFile(gzname, resname); } cmSystemTools::RemoveFile(gzname); } else { - this->RenameFile(this->TempName.c_str(), resname.c_str()); + this->RenameFile(this->TempName, resname); } replaced = true; @@ -188,10 +188,10 @@ bool cmGeneratedFileStreamBase::Close() } #ifdef CMAKE_BUILD_WITH_CMAKE -int cmGeneratedFileStreamBase::CompressFile(const char* oldname, - const char* newname) +int cmGeneratedFileStreamBase::CompressFile(std::string const& oldname, + std::string const& newname) { - gzFile gf = gzopen(newname, "w"); + gzFile gf = gzopen(newname.c_str(), "w"); if (!gf) { return 0; } @@ -214,16 +214,17 @@ int cmGeneratedFileStreamBase::CompressFile(const char* oldname, return 1; } #else -int cmGeneratedFileStreamBase::CompressFile(const char*, const char*) +int cmGeneratedFileStreamBase::CompressFile(std::string const&, + std::string const&) { return 0; } #endif -int cmGeneratedFileStreamBase::RenameFile(const char* oldname, - const char* newname) +int cmGeneratedFileStreamBase::RenameFile(std::string const& oldname, + std::string const& newname) { - return cmSystemTools::RenameFile(oldname, newname); + return cmSystemTools::RenameFile(oldname.c_str(), newname.c_str()); } void cmGeneratedFileStream::SetName(const std::string& fname) diff --git a/Source/cmGeneratedFileStream.h b/Source/cmGeneratedFileStream.h index 48f93c5..dacd166 100644 --- a/Source/cmGeneratedFileStream.h +++ b/Source/cmGeneratedFileStream.h @@ -20,7 +20,7 @@ protected: cmGeneratedFileStreamBase(); // This constructor prepares the temporary output file. - cmGeneratedFileStreamBase(const char* name); + cmGeneratedFileStreamBase(std::string const& name); // The destructor renames the temporary output file to the real name. ~cmGeneratedFileStreamBase(); @@ -29,14 +29,14 @@ protected: // called before the real stream is opened. Close is always called // after the real stream is closed and Okay is set to whether the // real stream was still valid for writing when it was closed. - void Open(const char* name); + void Open(std::string const& name); bool Close(); // Internal file replacement implementation. - int RenameFile(const char* oldname, const char* newname); + int RenameFile(std::string const& oldname, std::string const& newname); // Internal file compression implementation. - int CompressFile(const char* oldname, const char* newname); + int CompressFile(std::string const& oldname, std::string const& newname); // The name of the final destination file for the output. std::string Name; @@ -87,7 +87,7 @@ public: * file cannot be opened an error message is produced unless the * second argument is set to true. */ - cmGeneratedFileStream(const char* name, bool quiet = false, + cmGeneratedFileStream(std::string const& name, bool quiet = false, Encoding encoding = codecvt::None); /** @@ -103,7 +103,7 @@ public: * temporary file. If the file cannot be opened an error message is * produced unless the second argument is set to true. */ - cmGeneratedFileStream& Open(const char* name, bool quiet = false, + cmGeneratedFileStream& Open(std::string const& name, bool quiet = false, bool binaryFlag = false); /** diff --git a/Source/cmGeneratorExpressionEvaluationFile.cxx b/Source/cmGeneratorExpressionEvaluationFile.cxx index 99b9261..a094fa5 100644 --- a/Source/cmGeneratorExpressionEvaluationFile.cxx +++ b/Source/cmGeneratorExpressionEvaluationFile.cxx @@ -85,7 +85,7 @@ void cmGeneratorExpressionEvaluationFile::Generate( this->Files.push_back(outputFileName); outputFiles[outputFileName] = outputContent; - cmGeneratedFileStream fout(outputFileName.c_str()); + cmGeneratedFileStream fout(outputFileName); fout.SetCopyIfDifferent(true); fout << outputContent; if (fout.Close() && perm) { diff --git a/Source/cmGeneratorExpressionNode.cxx b/Source/cmGeneratorExpressionNode.cxx index 1594904..3381e6d 100644 --- a/Source/cmGeneratorExpressionNode.cxx +++ b/Source/cmGeneratorExpressionNode.cxx @@ -1126,7 +1126,7 @@ static const struct TargetPropertyNode : public cmGeneratorExpressionNode std::string targetName = parameters.front(); propertyName = parameters[1]; if (!cmGeneratorExpression::IsValidTargetName(targetName)) { - if (!propertyNameValidator.find(propertyName.c_str())) { + if (!propertyNameValidator.find(propertyName)) { ::reportError(context, content->GetOriginalExpression(), "Target name and property name not supported."); return std::string(); diff --git a/Source/cmGeneratorTarget.cxx b/Source/cmGeneratorTarget.cxx index 098c13a..c20fd13 100644 --- a/Source/cmGeneratorTarget.cxx +++ b/Source/cmGeneratorTarget.cxx @@ -1139,7 +1139,7 @@ void cmGeneratorTarget::ComputeKindedSources(KindedSources& files, std::string cppFileName = xaml + ".cpp"; files.ExpectedXamlHeaders.insert(hFileName); files.ExpectedXamlSources.insert(cppFileName); - } else if (header_regex.find(sf->GetFullPath().c_str())) { + } else if (header_regex.find(sf->GetFullPath())) { kind = SourceKindHeader; } else { kind = SourceKindExtra; diff --git a/Source/cmGlobVerificationManager.cxx b/Source/cmGlobVerificationManager.cxx index e23b6ea..d8c854c 100644 --- a/Source/cmGlobVerificationManager.cxx +++ b/Source/cmGlobVerificationManager.cxx @@ -23,7 +23,7 @@ bool cmGlobVerificationManager::SaveVerificationScript(const std::string& path) cmSystemTools::MakeDirectory(scriptFile); scriptFile += "/VerifyGlobs.cmake"; stampFile += "/cmake.verify_globs"; - cmGeneratedFileStream verifyScriptFile(scriptFile.c_str()); + cmGeneratedFileStream verifyScriptFile(scriptFile); verifyScriptFile.SetCopyIfDifferent(true); if (!verifyScriptFile) { cmSystemTools::Error("Unable to open verification script file for save. ", diff --git a/Source/cmGlobalGenerator.cxx b/Source/cmGlobalGenerator.cxx index 63bbf04..324b56a 100644 --- a/Source/cmGlobalGenerator.cxx +++ b/Source/cmGlobalGenerator.cxx @@ -1313,16 +1313,10 @@ bool cmGlobalGenerator::Compute() // so create the map from project name to vector of local generators this->FillProjectMap(); -#ifdef CMAKE_BUILD_WITH_CMAKE - // Iterate through all targets and set up automoc for those which have - // the AUTOMOC, AUTOUIC or AUTORCC property set - auto autogenInits = this->CreateQtAutoGenInitializers(); - for (auto& autoGen : autogenInits) { - if (!autoGen->InitCustomTargets()) { - return false; - } + // Iterate through all targets and set up AUTOMOC, AUTOUIC and AUTORCC + if (!this->QtAutoGen()) { + return false; } -#endif // Add generator specific helper commands for (cmLocalGenerator* localGen : this->LocalGenerators) { @@ -1341,16 +1335,6 @@ bool cmGlobalGenerator::Compute() } } -#ifdef CMAKE_BUILD_WITH_CMAKE - for (auto& autoGen : autogenInits) { - if (!autoGen->SetupCustomTargets()) { - return false; - } - autoGen.reset(nullptr); - } - autogenInits.clear(); -#endif - for (cmLocalGenerator* localGen : this->LocalGenerators) { cmMakefile* mf = localGen->GetMakefile(); for (cmInstallGenerator* g : mf->GetInstallGenerators()) { @@ -1480,12 +1464,11 @@ bool cmGlobalGenerator::ComputeTargetDepends() return true; } -std::vector<std::unique_ptr<cmQtAutoGenInitializer>> -cmGlobalGenerator::CreateQtAutoGenInitializers() +bool cmGlobalGenerator::QtAutoGen() { +#ifdef CMAKE_BUILD_WITH_CMAKE std::vector<std::unique_ptr<cmQtAutoGenInitializer>> autogenInits; -#ifdef CMAKE_BUILD_WITH_CMAKE for (cmLocalGenerator* localGen : this->LocalGenerators) { const std::vector<cmGeneratorTarget*>& targets = localGen->GetGeneratorTargets(); @@ -1519,12 +1502,30 @@ cmGlobalGenerator::CreateQtAutoGenInitializers() continue; } - autogenInits.emplace_back(new cmQtAutoGenInitializer( + autogenInits.emplace_back(cm::make_unique<cmQtAutoGenInitializer>( target, mocEnabled, uicEnabled, rccEnabled, qtVersionMajor)); } } + + if (!autogenInits.empty()) { + // Initialize custom targets + for (auto& autoGen : autogenInits) { + if (!autoGen->InitCustomTargets()) { + return false; + } + } + + // Setup custom targets + for (auto& autoGen : autogenInits) { + if (!autoGen->SetupCustomTargets()) { + return false; + } + autoGen.reset(nullptr); + } + } #endif - return autogenInits; + + return true; } cmLinkLineComputer* cmGlobalGenerator::CreateLinkLineComputer( @@ -2903,7 +2904,7 @@ void cmGlobalGenerator::WriteRuleHashes(std::string const& pfile) if (this->RuleHashes.empty()) { cmSystemTools::RemoveFile(pfile); } else { - cmGeneratedFileStream fout(pfile.c_str()); + cmGeneratedFileStream fout(pfile); fout << "# Hashes of file build rules.\n"; for (auto const& rh : this->RuleHashes) { fout.write(rh.second.Data, 32); @@ -2918,7 +2919,7 @@ void cmGlobalGenerator::WriteSummary() std::string fname = this->CMakeInstance->GetHomeOutputDirectory(); fname += cmake::GetCMakeFilesDirectory(); fname += "/TargetDirectories.txt"; - cmGeneratedFileStream fout(fname.c_str()); + cmGeneratedFileStream fout(fname); for (cmLocalGenerator* lg : this->LocalGenerators) { const std::vector<cmGeneratorTarget*>& tgts = lg->GetGeneratorTargets(); @@ -2955,7 +2956,7 @@ void cmGlobalGenerator::WriteSummary(cmGeneratorTarget* target) Json::Value& lj_sources = lj_root["sources"] = Json::arrayValue; cmSystemTools::MakeDirectory(dir); - cmGeneratedFileStream fout(file.c_str()); + cmGeneratedFileStream fout(file); std::vector<std::string> labels; @@ -3029,7 +3030,7 @@ void cmGlobalGenerator::WriteSummary(cmGeneratorTarget* target) } } } - cmGeneratedFileStream json_fout(json_file.c_str()); + cmGeneratedFileStream json_fout(json_file); json_fout << lj_root; } else #endif @@ -3104,7 +3105,7 @@ bool cmGlobalGenerator::GenerateCPackPropertiesFile() return true; } - cmGeneratedFileStream file(path.c_str()); + cmGeneratedFileStream file(path); file << "# CPack properties\n"; for (auto const& i : installedFiles) { diff --git a/Source/cmGlobalGenerator.h b/Source/cmGlobalGenerator.h index a50cc3b..1ed070e 100644 --- a/Source/cmGlobalGenerator.h +++ b/Source/cmGlobalGenerator.h @@ -33,7 +33,6 @@ class cmLinkLineComputer; class cmLocalGenerator; class cmMakefile; class cmOutputConverter; -class cmQtAutoGenInitializer; class cmSourceFile; class cmStateDirectory; class cmake; @@ -354,6 +353,8 @@ public: i.e. "Can I build Debug and Release in the same tree?" */ virtual bool IsMultiConfig() const { return false; } + virtual bool IsXcode() const { return false; } + /** Return true if we know the exact location of object files. If false, store the reason in the given string. This is meaningful only after EnableLanguage has been called. */ @@ -441,9 +442,9 @@ protected: virtual bool CheckALLOW_DUPLICATE_CUSTOM_TARGETS() const; - // Qt auto generators - std::vector<std::unique_ptr<cmQtAutoGenInitializer>> - CreateQtAutoGenInitializers(); + /// @brief Qt AUTOMOC/UIC/RCC target generation + /// @return true on success + bool QtAutoGen(); std::string SelectMakeProgram(const std::string& makeProgram, const std::string& makeDefault = "") const; diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index 6f7e82a..8ba38df 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -763,7 +763,7 @@ void cmGlobalNinjaGenerator::OpenBuildFileStream() // Get a stream where to generate things. if (!this->BuildFileStream) { this->BuildFileStream = new cmGeneratedFileStream( - buildFilePath.c_str(), false, this->GetMakefileEncoding()); + buildFilePath, false, this->GetMakefileEncoding()); if (!this->BuildFileStream) { // An error message is generated by the constructor if it cannot // open the file. @@ -801,7 +801,7 @@ void cmGlobalNinjaGenerator::OpenRulesFileStream() // Get a stream where to generate things. if (!this->RulesFileStream) { this->RulesFileStream = new cmGeneratedFileStream( - rulesFilePath.c_str(), false, this->GetMakefileEncoding()); + rulesFilePath, false, this->GetMakefileEncoding()); if (!this->RulesFileStream) { // An error message is generated by the constructor if it cannot // open the file. @@ -883,8 +883,7 @@ void cmGlobalNinjaGenerator::AddCXXCompileCommand( } // Get a stream where to generate things. - this->CompileCommandsStream = - new cmGeneratedFileStream(buildFilePath.c_str()); + this->CompileCommandsStream = new cmGeneratedFileStream(buildFilePath); *this->CompileCommandsStream << "["; } else { *this->CompileCommandsStream << "," << std::endl; @@ -1700,7 +1699,7 @@ int cmcmd_cmake_ninja_depends(std::vector<std::string>::const_iterator argBeg, } { - cmGeneratedFileStream depfile(arg_dep.c_str()); + cmGeneratedFileStream depfile(arg_dep); depfile << cmSystemTools::ConvertToUnixOutputPath(arg_pp) << ":"; for (std::string const& include : info.Includes) { depfile << " \\\n " << cmSystemTools::ConvertToUnixOutputPath(include); @@ -1723,7 +1722,7 @@ int cmcmd_cmake_ninja_depends(std::vector<std::string>::const_iterator argBeg, } } - cmGeneratedFileStream ddif(arg_ddi.c_str()); + cmGeneratedFileStream ddif(arg_ddi); ddif << ddi; if (!ddif) { cmSystemTools::Error("-E cmake_ninja_depends failed to write ", @@ -1827,7 +1826,7 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile( } } - cmGeneratedFileStream ddf(arg_dd.c_str()); + cmGeneratedFileStream ddf(arg_dd); ddf << "ninja_dyndep_version = 1.0\n"; for (cmFortranObjectInfo const& object : objects) { @@ -1862,7 +1861,7 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile( // use by dependents that reference this target in linked-target-dirs. std::string const target_mods_file = cmSystemTools::GetFilenamePath(arg_dd) + "/FortranModules.json"; - cmGeneratedFileStream tmf(target_mods_file.c_str()); + cmGeneratedFileStream tmf(target_mods_file); tmf << tm; return true; diff --git a/Source/cmGlobalUnixMakefileGenerator3.cxx b/Source/cmGlobalUnixMakefileGenerator3.cxx index 61c42be..db72353 100644 --- a/Source/cmGlobalUnixMakefileGenerator3.cxx +++ b/Source/cmGlobalUnixMakefileGenerator3.cxx @@ -147,7 +147,7 @@ void cmGlobalUnixMakefileGenerator3::Generate() markFileName += "/"; markFileName += cmake::GetCMakeFilesDirectory(); markFileName += "/progress.marks"; - cmGeneratedFileStream markFile(markFileName.c_str()); + cmGeneratedFileStream markFile(markFileName); markFile << this->CountProgressMarksInAll(lg) << "\n"; } @@ -170,8 +170,7 @@ void cmGlobalUnixMakefileGenerator3::AddCXXCompileCommand( std::string commandDatabaseName = std::string(this->GetCMakeInstance()->GetHomeOutputDirectory()) + "/compile_commands.json"; - this->CommandDatabase = - new cmGeneratedFileStream(commandDatabaseName.c_str()); + this->CommandDatabase = new cmGeneratedFileStream(commandDatabaseName); *this->CommandDatabase << "[" << std::endl; } else { *this->CommandDatabase << "," << std::endl; @@ -198,7 +197,7 @@ void cmGlobalUnixMakefileGenerator3::WriteMainMakefile2() this->GetCMakeInstance()->GetHomeOutputDirectory(); makefileName += cmake::GetCMakeFilesDirectory(); makefileName += "/Makefile2"; - cmGeneratedFileStream makefileStream(makefileName.c_str(), false, + cmGeneratedFileStream makefileStream(makefileName, false, this->GetMakefileEncoding()); if (!makefileStream) { return; @@ -270,7 +269,7 @@ void cmGlobalUnixMakefileGenerator3::WriteMainCMakefile() this->GetCMakeInstance()->GetHomeOutputDirectory(); cmakefileName += cmake::GetCMakeFilesDirectory(); cmakefileName += "/Makefile.cmake"; - cmGeneratedFileStream cmakefileStream(cmakefileName.c_str()); + cmGeneratedFileStream cmakefileStream(cmakefileName); if (!cmakefileStream) { return; } @@ -876,7 +875,7 @@ void cmGlobalUnixMakefileGenerator3::RecordTargetProgress( void cmGlobalUnixMakefileGenerator3::TargetProgress::WriteProgressVariables( unsigned long total, unsigned long& current) { - cmGeneratedFileStream fout(this->VariableFile.c_str()); + cmGeneratedFileStream fout(this->VariableFile); for (unsigned long i = 1; i <= this->NumberOfActions; ++i) { fout << "CMAKE_PROGRESS_" << i << " = "; if (total <= 100) { diff --git a/Source/cmGlobalXCodeGenerator.h b/Source/cmGlobalXCodeGenerator.h index f7bca13..62f7030 100644 --- a/Source/cmGlobalXCodeGenerator.h +++ b/Source/cmGlobalXCodeGenerator.h @@ -96,6 +96,8 @@ public: i.e. "Can I build Debug and Release in the same tree?" */ bool IsMultiConfig() const override; + bool IsXcode() const override { return true; } + bool HasKnownObjectFileLocation(std::string* reason) const override; bool IsIPOSupported() const override { return true; } diff --git a/Source/cmGraphVizWriter.cxx b/Source/cmGraphVizWriter.cxx index 1bbfbe0..fcdf03f 100644 --- a/Source/cmGraphVizWriter.cxx +++ b/Source/cmGraphVizWriter.cxx @@ -257,7 +257,7 @@ void cmGraphVizWriter::WriteTargetDependersFiles(const char* fileName) currentFilename += ptr.first; currentFilename += ".dependers"; - cmGeneratedFileStream str(currentFilename.c_str()); + cmGeneratedFileStream str(currentFilename); if (!str) { return; } @@ -300,7 +300,7 @@ void cmGraphVizWriter::WritePerTargetFiles(const char* fileName) std::string currentFilename = fileName; currentFilename += "."; currentFilename += ptr.first; - cmGeneratedFileStream str(currentFilename.c_str()); + cmGeneratedFileStream str(currentFilename); if (!str) { return; } diff --git a/Source/cmLocalGenerator.cxx b/Source/cmLocalGenerator.cxx index a669477..d265cab 100644 --- a/Source/cmLocalGenerator.cxx +++ b/Source/cmLocalGenerator.cxx @@ -253,7 +253,7 @@ void cmLocalGenerator::GenerateTestFiles() file += "/"; file += "CTestTestfile.cmake"; - cmGeneratedFileStream fout(file.c_str()); + cmGeneratedFileStream fout(file); fout.SetCopyIfDifferent(true); fout << "# CMake generated Testfile for " << std::endl @@ -429,7 +429,7 @@ void cmLocalGenerator::GenerateInstallRules() toplevel_install = 1; } file += "/cmake_install.cmake"; - cmGeneratedFileStream fout(file.c_str()); + cmGeneratedFileStream fout(file); fout.SetCopyIfDifferent(true); // Write the header. @@ -1971,7 +1971,7 @@ void cmLocalGenerator::AppendCompileOptions( cmsys::RegularExpression r(regex); for (std::string const& opt : options_vec) { - if (r.find(opt.c_str())) { + if (r.find(opt)) { this->AppendFlagEscape(options, opt); } } diff --git a/Source/cmLocalUnixMakefileGenerator3.cxx b/Source/cmLocalUnixMakefileGenerator3.cxx index 9fd2ddb..bc83ce2 100644 --- a/Source/cmLocalUnixMakefileGenerator3.cxx +++ b/Source/cmLocalUnixMakefileGenerator3.cxx @@ -211,8 +211,7 @@ void cmLocalUnixMakefileGenerator3::WriteLocalMakefile() // rules may depend on this file itself. std::string ruleFileNameFull = this->ConvertToFullPath(ruleFileName); cmGeneratedFileStream ruleFileStream( - ruleFileNameFull.c_str(), false, - this->GlobalGenerator->GetMakefileEncoding()); + ruleFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); if (!ruleFileStream) { return; } @@ -433,7 +432,7 @@ void cmLocalUnixMakefileGenerator3::WriteDirectoryInformationFile() infoFileName += "/CMakeDirectoryInformation.cmake"; // Open the output file. - cmGeneratedFileStream infoFileStream(infoFileName.c_str()); + cmGeneratedFileStream infoFileStream(infoFileName); if (!infoFileStream) { return; } @@ -1403,8 +1402,7 @@ bool cmLocalUnixMakefileGenerator3::ScanDependencies( std::string ruleFileNameFull = dir; ruleFileNameFull += "/depend.make"; cmGeneratedFileStream ruleFileStream( - ruleFileNameFull.c_str(), false, - this->GlobalGenerator->GetMakefileEncoding()); + ruleFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); ruleFileStream.SetCopyIfDifferent(true); if (!ruleFileStream) { return false; @@ -1416,7 +1414,7 @@ bool cmLocalUnixMakefileGenerator3::ScanDependencies( std::string internalRuleFileNameFull = dir; internalRuleFileNameFull += "/depend.internal"; cmGeneratedFileStream internalRuleFileStream( - internalRuleFileNameFull.c_str(), false, + internalRuleFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); if (!internalRuleFileStream) { return false; diff --git a/Source/cmMakefileTargetGenerator.cxx b/Source/cmMakefileTargetGenerator.cxx index 7cae305..79baca6 100644 --- a/Source/cmMakefileTargetGenerator.cxx +++ b/Source/cmMakefileTargetGenerator.cxx @@ -124,7 +124,7 @@ void cmMakefileTargetGenerator::CreateRuleFile() // Open the rule file. This should be copy-if-different because the // rules may depend on this file itself. this->BuildFileStream = - new cmGeneratedFileStream(this->BuildFileNameFull.c_str(), false, + new cmGeneratedFileStream(this->BuildFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); this->BuildFileStream->SetCopyIfDifferent(true); if (!this->BuildFileStream) { @@ -237,8 +237,7 @@ void cmMakefileTargetGenerator::WriteCommonCodeRules() if (!cmSystemTools::FileExists(dependFileNameFull)) { // Write an empty dependency file. cmGeneratedFileStream depFileStream( - dependFileNameFull.c_str(), false, - this->GlobalGenerator->GetMakefileEncoding()); + dependFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); depFileStream << "# Empty dependencies file for " << this->GeneratorTarget->GetName() << ".\n" << "# This may be replaced when dependencies are built." @@ -250,7 +249,7 @@ void cmMakefileTargetGenerator::WriteCommonCodeRules() this->FlagFileNameFull = this->TargetBuildDirectoryFull; this->FlagFileNameFull += "/flags.make"; this->FlagFileStream = - new cmGeneratedFileStream(this->FlagFileNameFull.c_str(), false, + new cmGeneratedFileStream(this->FlagFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); this->FlagFileStream->SetCopyIfDifferent(true); if (!this->FlagFileStream) { @@ -974,8 +973,7 @@ void cmMakefileTargetGenerator::WriteTargetDependRules() this->InfoFileNameFull += "/DependInfo.cmake"; this->InfoFileNameFull = this->LocalGenerator->ConvertToFullPath(this->InfoFileNameFull); - this->InfoFileStream = - new cmGeneratedFileStream(this->InfoFileNameFull.c_str()); + this->InfoFileStream = new cmGeneratedFileStream(this->InfoFileNameFull); this->InfoFileStream->SetCopyIfDifferent(true); if (!*this->InfoFileStream) { return; @@ -1446,7 +1444,7 @@ void cmMakefileTargetGenerator::CreateLinkScript( std::string linkScriptName = this->TargetBuildDirectoryFull; linkScriptName += "/"; linkScriptName += name; - cmGeneratedFileStream linkScriptStream(linkScriptName.c_str()); + cmGeneratedFileStream linkScriptStream(linkScriptName); linkScriptStream.SetCopyIfDifferent(true); for (std::string const& link_command : link_commands) { // Do not write out empty commands or commands beginning in the @@ -1530,7 +1528,7 @@ std::string cmMakefileTargetGenerator::CreateResponseFile( std::string responseFileNameFull = this->TargetBuildDirectoryFull; responseFileNameFull += "/"; responseFileNameFull += name; - cmGeneratedFileStream responseStream(responseFileNameFull.c_str()); + cmGeneratedFileStream responseStream(responseFileNameFull); responseStream.SetCopyIfDifferent(true); responseStream << options << "\n"; @@ -1718,7 +1716,7 @@ void cmMakefileTargetGenerator::GenDefFile( cmOutputConverter::SHELL); real_link_commands.insert(real_link_commands.begin(), cmd); // create a list of obj files for the -E __create_def to read - cmGeneratedFileStream fout(objlist_file.c_str()); + cmGeneratedFileStream fout(objlist_file); if (mdi->WindowsExportAllSymbols) { for (std::string const& obj : this->Objects) { diff --git a/Source/cmNinjaNormalTargetGenerator.cxx b/Source/cmNinjaNormalTargetGenerator.cxx index f94e5bc..b634b0a 100644 --- a/Source/cmNinjaNormalTargetGenerator.cxx +++ b/Source/cmNinjaNormalTargetGenerator.cxx @@ -956,7 +956,7 @@ void cmNinjaNormalTargetGenerator::WriteLinkStatement() preLinkCmdLines.push_back(std::move(cmd)); // create a list of obj files for the -E __create_def to read - cmGeneratedFileStream fout(obj_list_file.c_str()); + cmGeneratedFileStream fout(obj_list_file); if (mdi->WindowsExportAllSymbols) { cmNinjaDeps objs = this->GetObjects(); diff --git a/Source/cmNinjaTargetGenerator.cxx b/Source/cmNinjaTargetGenerator.cxx index df32f40..1cfaac5 100644 --- a/Source/cmNinjaTargetGenerator.cxx +++ b/Source/cmNinjaTargetGenerator.cxx @@ -1133,7 +1133,7 @@ void cmNinjaTargetGenerator::WriteTargetDependInfo(std::string const& lang) } std::string const tdin = this->GetTargetDependInfoPath(lang); - cmGeneratedFileStream tdif(tdin.c_str()); + cmGeneratedFileStream tdif(tdin); tdif << tdi; } diff --git a/Source/cmOutputRequiredFilesCommand.cxx b/Source/cmOutputRequiredFilesCommand.cxx index bdb98ca..e2de3f9 100644 --- a/Source/cmOutputRequiredFilesCommand.cxx +++ b/Source/cmOutputRequiredFilesCommand.cxx @@ -356,8 +356,7 @@ protected: if (!found) { // Couldn't find any dependency information. - if (this->ComplainFileRegularExpression.find( - info->IncludeName.c_str())) { + if (this->ComplainFileRegularExpression.find(info->IncludeName)) { cmSystemTools::Error("error cannot find dependencies for ", path); } else { // Destroy the name of the file so that it won't be output as a diff --git a/Source/cmQtAutoGenInitializer.cxx b/Source/cmQtAutoGenInitializer.cxx index 7cfdeba..72ae09a 100644 --- a/Source/cmQtAutoGenInitializer.cxx +++ b/Source/cmQtAutoGenInitializer.cxx @@ -1076,7 +1076,7 @@ bool cmQtAutoGenInitializer::SetupWriteAutogenInfo() cmGeneratedFileStream ofs; ofs.SetCopyIfDifferent(true); - ofs.Open(this->AutogenInfoFile.c_str(), false, true); + ofs.Open(this->AutogenInfoFile, false, true); if (ofs) { // Utility lambdas auto CWrite = [&ofs](const char* key, std::string const& value) { @@ -1183,7 +1183,7 @@ bool cmQtAutoGenInitializer::SetupWriteRccInfo() for (Qrc const& qrc : this->Rcc.Qrcs) { cmGeneratedFileStream ofs; ofs.SetCopyIfDifferent(true); - ofs.Open(qrc.InfoFile.c_str(), false, true); + ofs.Open(qrc.InfoFile, false, true); if (ofs) { // Utility lambdas auto CWrite = [&ofs](const char* key, std::string const& value) { diff --git a/Source/cmServerProtocol.cxx b/Source/cmServerProtocol.cxx index b05eac4..c267160 100644 --- a/Source/cmServerProtocol.cxx +++ b/Source/cmServerProtocol.cxx @@ -37,7 +37,6 @@ #include <set> #include <string> #include <unordered_map> -#include <utility> #include <vector> // Get rid of some windows macros: diff --git a/Source/cmStringCommand.cxx b/Source/cmStringCommand.cxx index ed88a1e..1605fd7 100644 --- a/Source/cmStringCommand.cxx +++ b/Source/cmStringCommand.cxx @@ -273,7 +273,7 @@ bool cmStringCommand::RegexMatch(std::vector<std::string> const& args) // Scan through the input for all matches. std::string output; - if (re.find(input.c_str())) { + if (re.find(input)) { this->Makefile->StoreMatches(re); std::string::size_type l = re.start(); std::string::size_type r = re.end(); diff --git a/Source/cmTarget.cxx b/Source/cmTarget.cxx index 803a0a9..380ac88 100644 --- a/Source/cmTarget.cxx +++ b/Source/cmTarget.cxx @@ -278,6 +278,31 @@ cmTarget::cmTarget(std::string const& name, cmStateEnums::TargetType type, this->SetPropertyDefault("LINK_SEARCH_START_STATIC", nullptr); this->SetPropertyDefault("LINK_SEARCH_END_STATIC", nullptr); this->SetPropertyDefault("FOLDER", nullptr); +#ifdef __APPLE__ + if (this->GetGlobalGenerator()->IsXcode()) { + this->SetPropertyDefault("XCODE_SCHEME_ADDRESS_SANITIZER", nullptr); + this->SetPropertyDefault( + "XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN", nullptr); + this->SetPropertyDefault("XCODE_SCHEME_THREAD_SANITIZER", nullptr); + this->SetPropertyDefault("XCODE_SCHEME_THREAD_SANITIZER_STOP", nullptr); + this->SetPropertyDefault("XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER", + nullptr); + this->SetPropertyDefault( + "XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP", nullptr); + this->SetPropertyDefault("XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER", + nullptr); + this->SetPropertyDefault("XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP", + nullptr); + this->SetPropertyDefault("XCODE_SCHEME_MALLOC_SCRIBBLE", nullptr); + this->SetPropertyDefault("XCODE_SCHEME_MALLOC_GUARD_EDGES", nullptr); + this->SetPropertyDefault("XCODE_SCHEME_GUARD_MALLOC", nullptr); + this->SetPropertyDefault("XCODE_SCHEME_ZOMBIE_OBJECTS", nullptr); + this->SetPropertyDefault("XCODE_SCHEME_MALLOC_STACK", nullptr); + this->SetPropertyDefault("XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE", + nullptr); + this->SetPropertyDefault("XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS", nullptr); + } +#endif } // Collect the set of configuration types. diff --git a/Source/cmUseMangledMesaCommand.cxx b/Source/cmUseMangledMesaCommand.cxx index ea012f6..01ef5cb 100644 --- a/Source/cmUseMangledMesaCommand.cxx +++ b/Source/cmUseMangledMesaCommand.cxx @@ -82,12 +82,12 @@ void cmUseMangledMesaCommand::CopyAndFullPathMesaHeader(const char* source, // regular expression for gl GL or xmesa in a file (match(1) of above) cmsys::RegularExpression glLine("(gl|GL|xmesa)"); while (cmSystemTools::GetLineFromStream(fin, inLine)) { - if (includeLine.find(inLine.c_str())) { + if (includeLine.find(inLine)) { std::string includeFile = includeLine.match(1); - if (glDirLine.find(includeFile.c_str())) { + if (glDirLine.find(includeFile)) { std::string gfile = glDirLine.match(3); fout << "#include \"" << outdir << "/" << gfile << "\"\n"; - } else if (glLine.find(includeFile.c_str())) { + } else if (glLine.find(includeFile)) { fout << "#include \"" << outdir << "/" << includeLine.match(1) << "\"\n"; } else { diff --git a/Source/cmWriteFileCommand.cxx b/Source/cmWriteFileCommand.cxx index d3d2db0..c504ef4 100644 --- a/Source/cmWriteFileCommand.cxx +++ b/Source/cmWriteFileCommand.cxx @@ -45,16 +45,20 @@ bool cmWriteFileCommand::InitialPass(std::vector<std::string> const& args, cmSystemTools::MakeDirectory(dir); mode_t mode = 0; + bool writable = false; // Set permissions to writable if (cmSystemTools::GetPermissions(fileName.c_str(), mode)) { - cmSystemTools::SetPermissions(fileName.c_str(), #if defined(_MSC_VER) || defined(__MINGW32__) - mode | S_IWRITE + writable = mode & S_IWRITE; + mode_t newMode = mode | S_IWRITE; #else - mode | S_IWUSR | S_IWGRP + writable = mode & S_IWUSR; + mode_t newMode = mode | S_IWUSR | S_IWGRP; #endif - ); + if (!writable) { + cmSystemTools::SetPermissions(fileName.c_str(), newMode); + } } // If GetPermissions fails, pretend like it is ok. File open will fail if // the file is not writable @@ -69,7 +73,7 @@ bool cmWriteFileCommand::InitialPass(std::vector<std::string> const& args, } file << message << std::endl; file.close(); - if (mode) { + if (mode && !writable) { cmSystemTools::SetPermissions(fileName.c_str(), mode); } diff --git a/Source/cmXCodeScheme.cxx b/Source/cmXCodeScheme.cxx index f1dce64..58cb9c9 100644 --- a/Source/cmXCodeScheme.cxx +++ b/Source/cmXCodeScheme.cxx @@ -143,6 +143,41 @@ void cmXCodeScheme::WriteLaunchAction(cmXMLWriter& xout, xout.Attribute("debugServiceExtension", "internal"); xout.Attribute("allowLocationSimulation", "YES"); + // Diagnostics tab begin + + bool useAddressSanitizer = WriteLaunchActionAttribute( + xout, "enableAddressSanitizer", + "XCODE_SCHEME_ADDRESS_SANITIZER"); // not allowed with + // enableThreadSanitizer=YES + WriteLaunchActionAttribute( + xout, "enableASanStackUseAfterReturn", + "XCODE_SCHEME_ADDRESS_SANITIZER_USE_AFTER_RETURN"); + + bool useThreadSanitizer = false; + if (!useAddressSanitizer) { + useThreadSanitizer = WriteLaunchActionAttribute( + xout, "enableThreadSanitizer", + "XCODE_SCHEME_THREAD_SANITIZER"); // not allowed with + // enableAddressSanitizer=YES + } + + WriteLaunchActionAttribute(xout, "stopOnEveryThreadSanitizerIssue", + "XCODE_SCHEME_THREAD_SANITIZER_STOP"); + + WriteLaunchActionAttribute(xout, "enableUBSanitizer", + "XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER"); + WriteLaunchActionAttribute( + xout, "stopOnEveryUBSanitizerIssue", + "XCODE_SCHEME_UNDEFINED_BEHAVIOUR_SANITIZER_STOP"); + + WriteLaunchActionAttribute( + xout, "disableMainThreadChecker", + "XCODE_SCHEME_DISABLE_MAIN_THREAD_CHECKER"); // negative enabled! + WriteLaunchActionAttribute(xout, "stopOnEveryMainThreadCheckerIssue", + "XCODE_SCHEME_MAIN_THREAD_CHECKER_STOP"); + + // Diagnostics tab end + if (IsExecutable(this->Target)) { xout.StartElement("BuildableProductRunnable"); xout.BreakAttributes(); @@ -156,12 +191,144 @@ void cmXCodeScheme::WriteLaunchAction(cmXMLWriter& xout, xout.EndElement(); // MacroExpansion + // Info tab begin + + if (const char* exe = + this->Target->GetTarget()->GetProperty("XCODE_SCHEME_EXECUTABLE")) { + + xout.StartElement("PathRunnable"); + xout.BreakAttributes(); + + xout.Attribute("runnableDebuggingMode", "0"); + xout.Attribute("FilePath", exe); + + xout.EndElement(); // PathRunnable + } + + // Info tab end + + // Arguments tab begin + + if (const char* argList = + this->Target->GetTarget()->GetProperty("XCODE_SCHEME_ARGUMENTS")) { + std::vector<std::string> arguments; + cmSystemTools::ExpandListArgument(argList, arguments); + if (!arguments.empty()) { + xout.StartElement("CommandLineArguments"); + + for (auto argument : arguments) { + xout.StartElement("CommandLineArgument"); + xout.BreakAttributes(); + + xout.Attribute("argument", argument); + xout.Attribute("isEnabled", "YES"); + + xout.EndElement(); // CommandLineArgument + } + + xout.EndElement(); // CommandLineArguments + } + } + + if (const char* envList = + this->Target->GetTarget()->GetProperty("XCODE_SCHEME_ENVIRONMENT")) { + std::vector<std::string> envs; + cmSystemTools::ExpandListArgument(envList, envs); + if (!envs.empty()) { + xout.StartElement("EnvironmentVariables"); + + for (auto env : envs) { + + xout.StartElement("EnvironmentVariable"); + xout.BreakAttributes(); + + std::string envValue; + const auto p = env.find_first_of('='); + if (p != std::string::npos) { + envValue = env.substr(p + 1); + env.resize(p); + } + + xout.Attribute("key", env); + xout.Attribute("value", envValue); + xout.Attribute("isEnabled", "YES"); + + xout.EndElement(); // EnvironmentVariable + } + + xout.EndElement(); // EnvironmentVariables + } + } + + // Arguments tab end + xout.StartElement("AdditionalOptions"); + + if (!useThreadSanitizer) { + WriteLaunchActionAdditionalOption(xout, "MallocScribble", "", + "XCODE_SCHEME_MALLOC_SCRIBBLE"); + } + + if (!useThreadSanitizer && !useAddressSanitizer) { + WriteLaunchActionAdditionalOption(xout, "MallocGuardEdges", "", + "XCODE_SCHEME_MALLOC_GUARD_EDGES"); + } + + if (!useThreadSanitizer && !useAddressSanitizer) { + WriteLaunchActionAdditionalOption(xout, "DYLD_INSERT_LIBRARIES", + "/usr/lib/libgmalloc.dylib", + "XCODE_SCHEME_GUARD_MALLOC"); + } + + WriteLaunchActionAdditionalOption(xout, "NSZombieEnabled", "YES", + "XCODE_SCHEME_ZOMBIE_OBJECTS"); + + if (!useThreadSanitizer && !useAddressSanitizer) { + WriteLaunchActionAdditionalOption(xout, "MallocStackLogging", "", + "XCODE_SCHEME_MALLOC_STACK"); + } + + WriteLaunchActionAdditionalOption(xout, "DYLD_PRINT_APIS", "", + "XCODE_SCHEME_DYNAMIC_LINKER_API_USAGE"); + + WriteLaunchActionAdditionalOption(xout, "DYLD_PRINT_LIBRARIES", "", + "XCODE_SCHEME_DYNAMIC_LIBRARY_LOADS"); + xout.EndElement(); xout.EndElement(); // LaunchAction } +bool cmXCodeScheme::WriteLaunchActionAttribute(cmXMLWriter& xout, + const std::string& attrName, + const std::string& varName) +{ + if (Target->GetTarget()->GetPropertyAsBool(varName)) { + xout.Attribute(attrName.c_str(), "YES"); + return true; + } + return false; +} + +bool cmXCodeScheme::WriteLaunchActionAdditionalOption( + cmXMLWriter& xout, const std::string& key, const std::string& value, + const std::string& varName) +{ + if (Target->GetTarget()->GetPropertyAsBool(varName)) { + xout.StartElement("AdditionalOption"); + xout.BreakAttributes(); + + xout.Attribute("key", key); + xout.Attribute("value", value); + xout.Attribute("isEnabled", "YES"); + + xout.EndElement(); // AdditionalOption + + return true; + } + return false; +} + void cmXCodeScheme::WriteProfileAction(cmXMLWriter& xout, const std::string& configuration) { diff --git a/Source/cmXCodeScheme.h b/Source/cmXCodeScheme.h index e5e501a..96c76e6 100644 --- a/Source/cmXCodeScheme.h +++ b/Source/cmXCodeScheme.h @@ -41,6 +41,16 @@ private: const std::string& container); void WriteLaunchAction(cmXMLWriter& xout, const std::string& configuration, const std::string& container); + + bool WriteLaunchActionAttribute(cmXMLWriter& xout, + const std::string& attrName, + const std::string& varName); + + bool WriteLaunchActionAdditionalOption(cmXMLWriter& xout, + const std::string& attrName, + const std::string& value, + const std::string& varName); + void WriteProfileAction(cmXMLWriter& xout, const std::string& configuration); void WriteAnalyzeAction(cmXMLWriter& xout, const std::string& configuration); void WriteArchiveAction(cmXMLWriter& xout, const std::string& configuration); diff --git a/Source/cmake.cxx b/Source/cmake.cxx index 6edaa74..5bb1f22 100644 --- a/Source/cmake.cxx +++ b/Source/cmake.cxx @@ -407,7 +407,7 @@ bool cmake::SetCacheArgs(const std::vector<std::string>& args) for (std::string const& ck : cacheKeys) { cmStateEnums::CacheEntryType t = this->State->GetCacheEntryType(ck); if (t != cmStateEnums::STATIC) { - if (regex.find(ck.c_str())) { + if (regex.find(ck)) { entriesToDelete.push_back(ck); } } diff --git a/Source/kwsys/SystemTools.cxx b/Source/kwsys/SystemTools.cxx index d552d9f..476fe08 100644 --- a/Source/kwsys/SystemTools.cxx +++ b/Source/kwsys/SystemTools.cxx @@ -1197,9 +1197,27 @@ bool SystemTools::FileExists(const std::string& filename) } return access(filename.c_str(), R_OK) == 0; #elif defined(_WIN32) - return ( - GetFileAttributesW(Encoding::ToWindowsExtendedPath(filename).c_str()) != - INVALID_FILE_ATTRIBUTES); + DWORD attr = + GetFileAttributesW(Encoding::ToWindowsExtendedPath(filename).c_str()); + if (attr == INVALID_FILE_ATTRIBUTES) { + return false; + } + + if (attr & FILE_ATTRIBUTE_REPARSE_POINT) { + // Using 0 instead of GENERIC_READ as it allows reading of file attributes + // even if we do not have permission to read the file itself + HANDLE handle = + CreateFileW(Encoding::ToWindowsExtendedPath(filename).c_str(), 0, 0, + NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + + if (handle == INVALID_HANDLE_VALUE) { + return false; + } + + CloseHandle(handle); + } + + return true; #else // SCO OpenServer 5.0.7/3.2's command has 711 permission. # if defined(_SCO_DS) diff --git a/Tests/QtAutogen/AutogenTest.cmake b/Tests/QtAutogen/AutogenTest.cmake index 8c0a14f..3969a89 100644 --- a/Tests/QtAutogen/AutogenTest.cmake +++ b/Tests/QtAutogen/AutogenTest.cmake @@ -6,7 +6,7 @@ if(QT_QMAKE_EXECUTABLE) list(APPEND CMAKE_PREFIX_PATH ${Qt_PREFIX_DIR}) endif() -if (QT_TEST_VERSION STREQUAL 4) +if (QT_TEST_VERSION EQUAL 4) find_package(Qt4 REQUIRED) include(UseQt4) @@ -21,7 +21,7 @@ if (QT_TEST_VERSION STREQUAL 4) qt4_generate_moc(${ARGN}) endmacro() -elseif(QT_TEST_VERSION STREQUAL 5) +elseif(QT_TEST_VERSION EQUAL 5) find_package(Qt5Widgets REQUIRED) set(QT_QTCORE_TARGET Qt5::Core) diff --git a/Tests/QtAutogen/CommonTests.cmake b/Tests/QtAutogen/CommonTests.cmake index 55d36e6..01ed7e9 100644 --- a/Tests/QtAutogen/CommonTests.cmake +++ b/Tests/QtAutogen/CommonTests.cmake @@ -12,7 +12,7 @@ if(QT_TEST_ALLOW_QT_MACROS) endif() ADD_AUTOGEN_TEST(UicSkipSource) ADD_AUTOGEN_TEST(RccSkipSource) -if(NOT QT_TEST_VERSION STREQUAL 4) +if(QT_TEST_VERSION GREATER 4) ADD_AUTOGEN_TEST(MocMacroName mocMacroName) endif() ADD_AUTOGEN_TEST(MocDepends) @@ -23,13 +23,13 @@ endif() if(QT_TEST_ALLOW_QT_MACROS) ADD_AUTOGEN_TEST(MocCMP0071) endif() -if(NOT QT_TEST_VERSION STREQUAL 4) - ADD_AUTOGEN_TEST(MocOsMacros mocOsMacros) +if(QT_TEST_VERSION GREATER 4) + ADD_AUTOGEN_TEST(MocOsMacros) endif() ADD_AUTOGEN_TEST(UicInclude uicInclude) ADD_AUTOGEN_TEST(UicInterface QtAutoUicInterface) ADD_AUTOGEN_TEST(ObjectLibrary someProgram) -if(APPLE AND (NOT QT_TEST_VERSION STREQUAL 4)) +if(APPLE AND (QT_TEST_VERSION GREATER 4)) ADD_AUTOGEN_TEST(MacOsFW) endif() ADD_AUTOGEN_TEST(Parallel parallel) @@ -42,7 +42,7 @@ ADD_AUTOGEN_TEST(SameName sameName) ADD_AUTOGEN_TEST(StaticLibraryCycle slc) # Rerun tests ADD_AUTOGEN_TEST(RerunMocBasic) -if(NOT QT_TEST_VERSION STREQUAL 4) +if(QT_TEST_VERSION GREATER 4) ADD_AUTOGEN_TEST(RerunMocPlugin) endif() ADD_AUTOGEN_TEST(RerunRccDepends) diff --git a/Tests/QtAutogen/DefinesTest/CMakeLists.txt b/Tests/QtAutogen/DefinesTest/CMakeLists.txt index de22845..3761dd8 100644 --- a/Tests/QtAutogen/DefinesTest/CMakeLists.txt +++ b/Tests/QtAutogen/DefinesTest/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.10) project(DefinesTest) # Qt4 only definitions test -if(NOT QT_TEST_VERSION STREQUAL 4) +if(NOT QT_TEST_VERSION EQUAL 4) message(ERROR "Invalid Qt test version. This test is for Qt4 only.") endif() diff --git a/Tests/QtAutogen/MocOsMacros/CMakeLists.txt b/Tests/QtAutogen/MocOsMacros/CMakeLists.txt index 51a0ebd..e7b670e 100644 --- a/Tests/QtAutogen/MocOsMacros/CMakeLists.txt +++ b/Tests/QtAutogen/MocOsMacros/CMakeLists.txt @@ -2,15 +2,31 @@ cmake_minimum_required(VERSION 3.11) project(MocOsMacros) include("../AutogenTest.cmake") -# Tests if moc processes the macro Q_OS_MAC +# Tests if moc processes Q_OS_XXX macros -set(CMAKE_AUTOMOC True) +message( "Qt5Core_VERSION: ${Qt5Core_VERSION}" ) +message( + "CMAKE_CXX_COMPILER_PREDEFINES_COMMAND: " + ${CMAKE_CXX_COMPILER_PREDEFINES_COMMAND} ) -add_executable(mocOsMacros - main.cpp - TestClass.cpp - TestClass.hpp +# On some platforms (e.g. MAC) Q_OS_XXX requires moc to include moc_predefs.h +# which is supported since Qt 5.8 and requires +# CMAKE_CXX_COMPILER_PREDEFINES_COMMAND to be defined. +if( ( ${Qt5Core_VERSION} VERSION_GREATER_EQUAL "5.8" ) AND + DEFINED CMAKE_CXX_COMPILER_PREDEFINES_COMMAND ) -target_link_libraries(mocOsMacros PRIVATE ${QT_QTCORE_TARGET}) + message( "Test enabled!" ) + message( + "CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES: " + ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES} ) -message ( "CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES: ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES}" ) + set(CMAKE_AUTOMOC True) + add_executable(mocOsMacros + main.cpp + TestClass.cpp + TestClass.hpp + ) + target_link_libraries(mocOsMacros PRIVATE ${QT_QTCORE_TARGET}) +else() + message( "Test disabled!" ) +endif() diff --git a/Tests/QtAutogen/RerunMocPlugin/MocPlugin/CMakeLists.txt b/Tests/QtAutogen/RerunMocPlugin/MocPlugin/CMakeLists.txt index bc0085f..ca22aeb 100644 --- a/Tests/QtAutogen/RerunMocPlugin/MocPlugin/CMakeLists.txt +++ b/Tests/QtAutogen/RerunMocPlugin/MocPlugin/CMakeLists.txt @@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.10) project(MocPlugin) include("../../AutogenTest.cmake") -if (NOT QT_TEST_VERSION STREQUAL 5) - message(SEND_ERROR "Invalid Qt version specified.") +if (QT_TEST_VERSION LESS 5) + message(SEND_ERROR "Qt 5 or higher required.") endif() set(CMAKE_AUTOMOC_DEPEND_FILTERS diff --git a/Tests/QtAutogen/SameName/CMakeLists.txt b/Tests/QtAutogen/SameName/CMakeLists.txt index c7d6e52..931e40f 100644 --- a/Tests/QtAutogen/SameName/CMakeLists.txt +++ b/Tests/QtAutogen/SameName/CMakeLists.txt @@ -28,7 +28,7 @@ set_target_properties(sameName PROPERTIES ) # Set different compression levels -if (QT_TEST_VERSION STREQUAL 4) +if (QT_TEST_VERSION EQUAL 4) set(rccCompress "-compress") else() set(rccCompress "--compress") diff --git a/Tests/QtAutogen/TestMacros.cmake b/Tests/QtAutogen/TestMacros.cmake index 8e4bea2..0e27188 100644 --- a/Tests/QtAutogen/TestMacros.cmake +++ b/Tests/QtAutogen/TestMacros.cmake @@ -50,7 +50,7 @@ if(NON_ASCII_BDIR) # Qt4 moc does not support utf8 paths in _parameter files generated by # qtx_wrap_cpp # https://bugreports.qt.io/browse/QTBUG-35480 - if(QT_TEST_VERSION STREQUAL 4) + if(QT_TEST_VERSION EQUAL 4) set(QT_TEST_ALLOW_QT_MACROS FALSE) endif() # On windows qtx_wrap_cpp also fails in Qt5 when used on a path that diff --git a/Tests/RunCMake/Syntax/QueryCache-stderr.txt b/Tests/RunCMake/Syntax/QueryCache-stderr.txt new file mode 100644 index 0000000..db6b2b7 --- /dev/null +++ b/Tests/RunCMake/Syntax/QueryCache-stderr.txt @@ -0,0 +1,2 @@ +-->cache value<-- +-->local value<-- diff --git a/Tests/RunCMake/Syntax/QueryCache.cmake b/Tests/RunCMake/Syntax/QueryCache.cmake new file mode 100644 index 0000000..20c648e --- /dev/null +++ b/Tests/RunCMake/Syntax/QueryCache.cmake @@ -0,0 +1,6 @@ + +set(query_var "cache value" CACHE STRING "") +set(query_var "local value") + +message("-->$CACHE{query_var}<--") +message("-->${query_var}<--") diff --git a/Tests/RunCMake/Syntax/RunCMakeTest.cmake b/Tests/RunCMake/Syntax/RunCMakeTest.cmake index b8f5fd0..a397620 100644 --- a/Tests/RunCMake/Syntax/RunCMakeTest.cmake +++ b/Tests/RunCMake/Syntax/RunCMakeTest.cmake @@ -112,6 +112,9 @@ run_cmake(CMP0053-NameWithCarriageReturnQuoted) run_cmake(CMP0053-NameWithEscapedSpacesQuoted) run_cmake(CMP0053-NameWithEscapedTabsQuoted) +# Variable special types +run_cmake(QueryCache) + # Function and macro tests. run_cmake(FunctionUnmatched) run_cmake(FunctionUnmatchedForeach) diff --git a/Tests/RunCMake/XcodeProject/RunCMakeTest.cmake b/Tests/RunCMake/XcodeProject/RunCMakeTest.cmake index 1150666..fb04005 100644 --- a/Tests/RunCMake/XcodeProject/RunCMakeTest.cmake +++ b/Tests/RunCMake/XcodeProject/RunCMakeTest.cmake @@ -219,6 +219,7 @@ endfunction() if(NOT XCODE_VERSION VERSION_LESS 7) XcodeSchemaGeneration() + run_cmake(XcodeSchemaProperty) endif() if(XCODE_VERSION VERSION_GREATER_EQUAL 8) diff --git a/Tests/RunCMake/XcodeProject/XcodeSchemaProperty-check.cmake b/Tests/RunCMake/XcodeProject/XcodeSchemaProperty-check.cmake new file mode 100644 index 0000000..f675d81 --- /dev/null +++ b/Tests/RunCMake/XcodeProject/XcodeSchemaProperty-check.cmake @@ -0,0 +1,33 @@ +function(check_property property matcher) + set(schema "${RunCMake_TEST_BINARY_DIR}/XcodeSchemaProperty.xcodeproj/xcshareddata/xcschemes/${property}.xcscheme") + file(STRINGS ${schema} actual-${property} + REGEX "${matcher}" LIMIT_COUNT 1) + if(NOT actual-${property}) + message(SEND_ERROR "Xcode schema property ${property}: Could not find ${matcher} in schema ${schema}") + endif() +endfunction() + +check_property("ADDRESS_SANITIZER" "enableAddressSanitizer") +check_property("ADDRESS_SANITIZER_USE_AFTER_RETURN" "enableASanStackUseAfterReturn") +check_property("THREAD_SANITIZER" "enableThreadSanitizer") +check_property("THREAD_SANITIZER_STOP" "stopOnEveryThreadSanitizerIssue") +check_property("UNDEFINED_BEHAVIOUR_SANITIZER" "enableUBSanitizer") +check_property("UNDEFINED_BEHAVIOUR_SANITIZER_STOP" "stopOnEveryUBSanitizerIssue") +check_property("DISABLE_MAIN_THREAD_CHECKER" "disableMainThreadChecker") +check_property("MAIN_THREAD_CHECKER_STOP" "stopOnEveryMainThreadCheckerIssue") + +check_property("MALLOC_SCRIBBLE" "MallocScribble") +check_property("MALLOC_GUARD_EDGES" "MallocGuardEdges") +check_property("GUARD_MALLOC" "DYLD_INSERT_LIBRARIES") +check_property("ZOMBIE_OBJECTS" "NSZombieEnabled") +check_property("MALLOC_STACK" "MallocStackLogging") +check_property("DYNAMIC_LINKER_API_USAGE" "DYLD_PRINT_APIS") +check_property("DYNAMIC_LIBRARY_LOADS" "DYLD_PRINT_LIBRARIES") + +check_property("EXECUTABLE" "myExecutable") +check_property("ARGUMENTS" [=["--foo"]=]) +check_property("ARGUMENTS" [=["--bar=baz"]=]) +check_property("ENVIRONMENT" [=[key="FOO"]=]) +check_property("ENVIRONMENT" [=[value="foo"]=]) +check_property("ENVIRONMENT" [=[key="BAR"]=]) +check_property("ENVIRONMENT" [=[value="bar"]=]) diff --git a/Tests/RunCMake/XcodeProject/XcodeSchemaProperty.cmake b/Tests/RunCMake/XcodeProject/XcodeSchemaProperty.cmake new file mode 100644 index 0000000..2b72a64 --- /dev/null +++ b/Tests/RunCMake/XcodeProject/XcodeSchemaProperty.cmake @@ -0,0 +1,37 @@ +cmake_minimum_required(VERSION 3.7) + +set(CMAKE_XCODE_GENERATE_SCHEME ON) + +project(XcodeSchemaProperty CXX) + +function(create_scheme_for_variable variable) + set(CMAKE_XCODE_SCHEME_${variable} ON) + add_executable(${variable} main.cpp) +endfunction() + +create_scheme_for_variable(ADDRESS_SANITIZER) +create_scheme_for_variable(ADDRESS_SANITIZER_USE_AFTER_RETURN) +create_scheme_for_variable(THREAD_SANITIZER) +create_scheme_for_variable(THREAD_SANITIZER_STOP) +create_scheme_for_variable(UNDEFINED_BEHAVIOUR_SANITIZER) +create_scheme_for_variable(UNDEFINED_BEHAVIOUR_SANITIZER_STOP) +create_scheme_for_variable(DISABLE_MAIN_THREAD_CHECKER) +create_scheme_for_variable(MAIN_THREAD_CHECKER_STOP) + +create_scheme_for_variable(MALLOC_SCRIBBLE) +create_scheme_for_variable(MALLOC_GUARD_EDGES) +create_scheme_for_variable(GUARD_MALLOC) +create_scheme_for_variable(ZOMBIE_OBJECTS) +create_scheme_for_variable(MALLOC_STACK) +create_scheme_for_variable(DYNAMIC_LINKER_API_USAGE) +create_scheme_for_variable(DYNAMIC_LIBRARY_LOADS) + +function(create_scheme_for_property property value) + set(XCODE_SCHEME_${property} ON) + add_executable(${property} main.cpp) + set_target_properties(${property} PROPERTIES XCODE_SCHEME_${property} "${value}") +endfunction() + +create_scheme_for_property(EXECUTABLE myExecutable) +create_scheme_for_property(ARGUMENTS "--foo;--bar=baz") +create_scheme_for_property(ENVIRONMENT "FOO=foo;BAR=bar") diff --git a/Utilities/IWYU/mapping.imp b/Utilities/IWYU/mapping.imp index 77bb572..5af8b85 100644 --- a/Utilities/IWYU/mapping.imp +++ b/Utilities/IWYU/mapping.imp @@ -53,7 +53,7 @@ #{ symbol: [ "std::pair", private, "<set>", public ] }, # HACK: iwyu wrongly thinks that <system_error> is needed for std::hash - { symbol: [ "std::hash", private, "<utility>", public ] }, + { symbol: [ "std::hash", private, "<functional>", public ] }, # __decay_and_strip is used internally in the C++11 standard library. # IWYU does not classify it as internal and suggests to add <type_traits>. diff --git a/Utilities/Scripts/update-liblzma.bash b/Utilities/Scripts/update-liblzma.bash index 088eb91..fdf66b3 100755 --- a/Utilities/Scripts/update-liblzma.bash +++ b/Utilities/Scripts/update-liblzma.bash @@ -7,8 +7,8 @@ shopt -s dotglob readonly name="liblzma" readonly ownership="liblzma upstream <xz-devel@tukaani.org>" readonly subtree="Utilities/cmliblzma" -readonly repo="http://git.tukaani.org/xz.git" -readonly tag="v5.0.8" +readonly repo="https://git.tukaani.org/xz.git" +readonly tag="v5.2.4" readonly shortlog=false readonly paths=" COPYING @@ -24,6 +24,10 @@ extract_source () { mv src/common . mv src/liblzma . rmdir src + rm liblzma/Makefile.* + rm liblzma/*/Makefile.* + rm liblzma/liblzma.map + rm liblzma/validate_map.sh popd } diff --git a/Utilities/cmliblzma/CMakeLists.txt b/Utilities/cmliblzma/CMakeLists.txt index bb3b8a7..e9f8826 100644 --- a/Utilities/cmliblzma/CMakeLists.txt +++ b/Utilities/cmliblzma/CMakeLists.txt @@ -6,22 +6,6 @@ include(CheckSymbolExists) include(CheckTypeSize) include(TestBigEndian) -CHECK_C_SOURCE_COMPILES( - "int test (void *restrict x);\nint main (void) {return 0;}" - HAVE_RESTRICT) - -CHECK_C_SOURCE_COMPILES( -"typedef struct abc *d;\nint test (d __restrict x);\nint main (void) {return 0;}" - HAVE___RESTRICT) - -CHECK_C_SOURCE_COMPILES( - "static inline int test (void) {return 0;}\nint main (void) {return test();}" - HAVE_INLINE) - -CHECK_C_SOURCE_COMPILES ( - "static __inline int test (void) {return 0;}\nint main (void) {return test();}" - HAVE___INLINE) - CHECK_INCLUDE_FILE(byteswap.h HAVE_BYTESWAP_H) CHECK_INCLUDE_FILE(inttypes.h HAVE_INTTYPES_H) CHECK_INCLUDE_FILE(limits.h HAVE_LIMITS_H) @@ -95,7 +79,7 @@ CHECK_TYPE_SIZE("unsigned short" SIZE_OF_UNSIGNED_SHORT) CHECK_TYPE_SIZE("unsigned" SIZE_OF_UNSIGNED) CHECK_TYPE_SIZE("unsigned long" SIZE_OF_UNSIGNED_LONG) CHECK_TYPE_SIZE("unsigned long long" SIZE_OF_UNSIGNED_LONG_LONG) -CHECK_TYPE_SIZE("size_t" SIZE_OF_SIZE_T) +CHECK_TYPE_SIZE("size_t" SIZEOF_SIZE_T) CHECK_TYPE_SIZE("__int64" __INT64) CHECK_TYPE_SIZE("unsigned __int64" UNSIGNED___INT64) diff --git a/Utilities/cmliblzma/COPYING b/Utilities/cmliblzma/COPYING index 43c90d0..20e60d5 100644 --- a/Utilities/cmliblzma/COPYING +++ b/Utilities/cmliblzma/COPYING @@ -47,7 +47,7 @@ XZ Utils Licensing naturally it is not legally required. Here is an example of a good notice to put into "about box" or into documentation: - This software includes code from XZ Utils <http://tukaani.org/xz/>. + This software includes code from XZ Utils <https://tukaani.org/xz/>. The following license texts are included in the following files: - COPYING.LGPLv2.1: GNU Lesser General Public License version 2.1 diff --git a/Utilities/cmliblzma/common/common_w32res.rc b/Utilities/cmliblzma/common/common_w32res.rc index fdb88d1..a70de34 100644 --- a/Utilities/cmliblzma/common/common_w32res.rc +++ b/Utilities/cmliblzma/common/common_w32res.rc @@ -17,7 +17,7 @@ #define MY_VERSION LZMA_VERSION_MAJOR,LZMA_VERSION_MINOR,LZMA_VERSION_PATCH,MY_BUILD #define MY_FILENAME MY_NAME MY_SUFFIX -#define MY_COMPANY "The Tukaani Project <http://tukaani.org/>" +#define MY_COMPANY "The Tukaani Project <https://tukaani.org/>" #define MY_PRODUCT PACKAGE_NAME " <" PACKAGE_URL ">" LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US diff --git a/Utilities/cmliblzma/common/sysdefs.h b/Utilities/cmliblzma/common/sysdefs.h index a6edea8..22f487b 100644 --- a/Utilities/cmliblzma/common/sysdefs.h +++ b/Utilities/cmliblzma/common/sysdefs.h @@ -18,6 +18,7 @@ #if defined(_MSC_VER) # pragma warning(push,1) +# pragma warning(disable: 4028) /* formal parameter different from decl */ # pragma warning(disable: 4142) /* benign redefinition of type */ # pragma warning(disable: 4761) /* integral size mismatch in argument */ #endif @@ -124,9 +125,9 @@ // The code currently assumes that size_t is either 32-bit or 64-bit. #ifndef SIZE_MAX -# if SIZE_OF_SIZE_T == 4 +# if SIZEOF_SIZE_T == 4 # define SIZE_MAX UINT32_MAX -# elif SIZE_OF_SIZE_T == 8 +# elif SIZEOF_SIZE_T == 8 # define SIZE_MAX UINT64_MAX # else # error size_t is not 32-bit or 64-bit @@ -175,6 +176,16 @@ typedef unsigned char _Bool; # include <memory.h> #endif +// As of MSVC 2013, inline and restrict are supported with +// non-standard keywords. +#if defined(_WIN32) && defined(_MSC_VER) +# ifndef inline +# define inline __inline +# endif +# ifndef restrict +# define restrict __restrict +# endif +#endif //////////// // Macros // diff --git a/Utilities/cmliblzma/common/tuklib_integer.h b/Utilities/cmliblzma/common/tuklib_integer.h index 5e8262a..5d82685 100644 --- a/Utilities/cmliblzma/common/tuklib_integer.h +++ b/Utilities/cmliblzma/common/tuklib_integer.h @@ -106,6 +106,17 @@ #endif +//////////////////////////////// +// Compiler-specific features // +//////////////////////////////// + +// Newer Intel C compilers require immintrin.h for _bit_scan_reverse() +// and such functions. +#if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1500) +# include <immintrin.h> +#endif + + /////////////////// // Byte swapping // /////////////////// @@ -329,8 +340,8 @@ unaligned_read32le(const uint8_t *buf) static inline void unaligned_write16be(uint8_t *buf, uint16_t num) { - buf[0] = num >> 8; - buf[1] = num; + buf[0] = (uint8_t)(num >> 8); + buf[1] = (uint8_t)num; return; } @@ -338,8 +349,8 @@ unaligned_write16be(uint8_t *buf, uint16_t num) static inline void unaligned_write16le(uint8_t *buf, uint16_t num) { - buf[0] = num; - buf[1] = num >> 8; + buf[0] = (uint8_t)num; + buf[1] = (uint8_t)(num >> 8); return; } @@ -347,10 +358,10 @@ unaligned_write16le(uint8_t *buf, uint16_t num) static inline void unaligned_write32be(uint8_t *buf, uint32_t num) { - buf[0] = num >> 24; - buf[1] = num >> 16; - buf[2] = num >> 8; - buf[3] = num; + buf[0] = (uint8_t)(num >> 24); + buf[1] = (uint8_t)(num >> 16); + buf[2] = (uint8_t)(num >> 8); + buf[3] = (uint8_t)num; return; } @@ -358,10 +369,10 @@ unaligned_write32be(uint8_t *buf, uint32_t num) static inline void unaligned_write32le(uint8_t *buf, uint32_t num) { - buf[0] = num; - buf[1] = num >> 8; - buf[2] = num >> 16; - buf[3] = num >> 24; + buf[0] = (uint8_t)num; + buf[1] = (uint8_t)(num >> 8); + buf[2] = (uint8_t)(num >> 16); + buf[3] = (uint8_t)(num >> 24); return; } diff --git a/Utilities/cmliblzma/config.h.in b/Utilities/cmliblzma/config.h.in index 9c53150..06f7fcb 100644 --- a/Utilities/cmliblzma/config.h.in +++ b/Utilities/cmliblzma/config.h.in @@ -29,7 +29,7 @@ @SIZE_OF_UNSIGNED_CODE@ @SIZE_OF_UNSIGNED_LONG_CODE@ @SIZE_OF_UNSIGNED_LONG_LONG_CODE@ -@SIZE_OF_SIZE_T_CODE@ +@SIZEOF_SIZE_T_CODE@ /* * If we lack int64_t, define it to the first of __int64, int, long, and long long @@ -180,32 +180,6 @@ typedef uint64_t uintmax_t; #cmakedefine uintptr_t @uintptr_t@ - -#cmakedefine HAVE_RESTRICT -#cmakedefine HAVE___RESTRICT - -#cmakedefine HAVE_INLINE -#cmakedefine HAVE___INLINE - -#ifndef HAVE_RESTRICT -# ifdef HAVE___RESTRICT -# define LZMA_RESTRICT __restrict -# else -# define LZMA_RESTRICT -# endif -#else -# define LZMA_RESTRICT restrict -#endif /* HAVE_RESTRICT */ - -#ifndef HAVE_INLINE -# ifdef HAVE___INLINE -# define inline __inline -# else -# define inline -# endif -#endif /* HAVE_INLINE */ - - #cmakedefine WORDS_BIGENDIAN 1 #cmakedefine HAVE_BYTESWAP_H 1 diff --git a/Utilities/cmliblzma/liblzma/api/lzma.h b/Utilities/cmliblzma/liblzma/api/lzma.h index fb874c3..aa88e42 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma.h +++ b/Utilities/cmliblzma/liblzma/api/lzma.h @@ -82,12 +82,20 @@ # if !defined(UINT32_C) || !defined(UINT64_C) \ || !defined(UINT32_MAX) || !defined(UINT64_MAX) /* - * MSVC has no C99 support, and thus it cannot be used to - * compile liblzma. The liblzma API has to still be usable - * from MSVC, so we need to define the required standard - * integer types here. + * MSVC versions older than 2013 have no C99 support, and + * thus they cannot be used to compile liblzma. Using an + * existing liblzma.dll with old MSVC can work though(*), + * but we need to define the required standard integer + * types here in a MSVC-specific way. + * + * (*) If you do this, the existing liblzma.dll probably uses + * a different runtime library than your MSVC-built + * application. Mixing runtimes is generally bad, but + * in this case it should work as long as you avoid + * the few rarely-needed liblzma functions that allocate + * memory and expect the caller to free it using free(). */ -# if defined(_WIN32) && defined(_MSC_VER) +# if defined(_WIN32) && defined(_MSC_VER) && _MSC_VER < 1800 typedef unsigned __int8 uint8_t; typedef unsigned __int32 uint32_t; typedef unsigned __int64 uint64_t; @@ -211,7 +219,11 @@ */ #ifndef lzma_nothrow # if defined(__cplusplus) -# define lzma_nothrow throw() +# if __cplusplus >= 201103L +# define lzma_nothrow noexcept +# else +# define lzma_nothrow throw() +# endif # elif __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 3) # define lzma_nothrow __attribute__((__nothrow__)) # else @@ -286,7 +298,7 @@ extern "C" { #include "lzma/filter.h" #include "lzma/bcj.h" #include "lzma/delta.h" -#include "lzma/lzma.h" +#include "lzma/lzma12.h" /* Container formats */ #include "lzma/container.h" diff --git a/Utilities/cmliblzma/liblzma/api/lzma/base.h b/Utilities/cmliblzma/liblzma/api/lzma/base.h index 43dde8d..a6005ac 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/base.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/base.h @@ -240,12 +240,12 @@ typedef enum { /** * \brief The `action' argument for lzma_code() * - * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, or LZMA_FINISH, - * the same `action' must is used until lzma_code() returns LZMA_STREAM_END. - * Also, the amount of input (that is, strm->avail_in) must not be modified - * by the application until lzma_code() returns LZMA_STREAM_END. Changing the - * `action' or modifying the amount of input will make lzma_code() return - * LZMA_PROG_ERROR. + * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, LZMA_FULL_BARRIER, + * or LZMA_FINISH, the same `action' must is used until lzma_code() returns + * LZMA_STREAM_END. Also, the amount of input (that is, strm->avail_in) must + * not be modified by the application until lzma_code() returns + * LZMA_STREAM_END. Changing the `action' or modifying the amount of input + * will make lzma_code() return LZMA_PROG_ERROR. */ typedef enum { LZMA_RUN = 0, @@ -293,7 +293,7 @@ typedef enum { * * All the input data going to the current Block must have * been given to the encoder (the last bytes can still be - * pending in* next_in). Call lzma_code() with LZMA_FULL_FLUSH + * pending in *next_in). Call lzma_code() with LZMA_FULL_FLUSH * until it returns LZMA_STREAM_END. Then continue normally * with LZMA_RUN or finish the Stream with LZMA_FINISH. * @@ -302,6 +302,29 @@ typedef enum { * no unfinished Block, no empty Block is created. */ + LZMA_FULL_BARRIER = 4, + /**< + * \brief Finish encoding of the current Block + * + * This is like LZMA_FULL_FLUSH except that this doesn't + * necessarily wait until all the input has been made + * available via the output buffer. That is, lzma_code() + * might return LZMA_STREAM_END as soon as all the input + * has been consumed (avail_in == 0). + * + * LZMA_FULL_BARRIER is useful with a threaded encoder if + * one wants to split the .xz Stream into Blocks at specific + * offsets but doesn't care if the output isn't flushed + * immediately. Using LZMA_FULL_BARRIER allows keeping + * the threads busy while LZMA_FULL_FLUSH would make + * lzma_code() wait until all the threads have finished + * until more data could be passed to the encoder. + * + * With a lzma_stream initialized with the single-threaded + * lzma_stream_encoder() or lzma_easy_encoder(), + * LZMA_FULL_BARRIER is an alias for LZMA_FULL_FLUSH. + */ + LZMA_FINISH = 3 /**< * \brief Finish the coding operation @@ -332,11 +355,19 @@ typedef enum { * malloc() and free(). C++ users should note that the custom memory * handling functions must not throw exceptions. * - * liblzma doesn't make an internal copy of lzma_allocator. Thus, it is - * OK to change these function pointers in the middle of the coding - * process, but obviously it must be done carefully to make sure that the - * replacement `free' can deallocate memory allocated by the earlier - * `alloc' function(s). + * Single-threaded mode only: liblzma doesn't make an internal copy of + * lzma_allocator. Thus, it is OK to change these function pointers in + * the middle of the coding process, but obviously it must be done + * carefully to make sure that the replacement `free' can deallocate + * memory allocated by the earlier `alloc' function(s). + * + * Multithreaded mode: liblzma might internally store pointers to the + * lzma_allocator given via the lzma_stream structure. The application + * must not change the allocator pointer in lzma_stream or the contents + * of the pointed lzma_allocator structure until lzma_end() has been used + * to free the memory associated with that lzma_stream. The allocation + * functions might be called simultaneously from multiple threads, and + * thus they must be thread safe. */ typedef struct { /** @@ -448,7 +479,8 @@ typedef struct lzma_internal_s lzma_internal; * * Application may modify the values of total_in and total_out as it wants. * They are updated by liblzma to match the amount of data read and - * written, but aren't used for anything else. + * written but aren't used for anything else except as a possible return + * values from lzma_get_progress(). */ typedef struct { const uint8_t *next_in; /**< Pointer to the next input byte. */ @@ -464,8 +496,10 @@ typedef struct { * * In most cases this is NULL which makes liblzma use * the standard malloc() and free(). + * + * \note In 5.0.x this is not a const pointer. */ - lzma_allocator *allocator; + const lzma_allocator *allocator; /** Internal state is not visible to applications. */ lzma_internal *internal; @@ -547,6 +581,25 @@ extern LZMA_API(void) lzma_end(lzma_stream *strm) lzma_nothrow; /** + * \brief Get progress information + * + * In single-threaded mode, applications can get progress information from + * strm->total_in and strm->total_out. In multi-threaded mode this is less + * useful because a significant amount of both input and output data gets + * buffered internally by liblzma. This makes total_in and total_out give + * misleading information and also makes the progress indicator updates + * non-smooth. + * + * This function gives realistic progress information also in multi-threaded + * mode by taking into account the progress made by each thread. In + * single-threaded mode *progress_in and *progress_out are set to + * strm->total_in and strm->total_out, respectively. + */ +extern LZMA_API(void) lzma_get_progress(lzma_stream *strm, + uint64_t *progress_in, uint64_t *progress_out) lzma_nothrow; + + +/** * \brief Get the memory usage of decoder filter chain * * This function is currently supported only when *strm has been initialized @@ -591,11 +644,16 @@ extern LZMA_API(uint64_t) lzma_memlimit_get(const lzma_stream *strm) * This function is supported only when *strm has been initialized with * a function that takes a memlimit argument. * + * liblzma 5.2.3 and earlier has a bug where memlimit value of 0 causes + * this function to do nothing (leaving the limit unchanged) and still + * return LZMA_OK. Later versions treat 0 as if 1 had been specified (so + * lzma_memlimit_get() will return 1 even if you specify 0 here). + * * \return - LZMA_OK: New memory usage limit successfully set. * - LZMA_MEMLIMIT_ERROR: The new limit is too small. * The limit was not changed. * - LZMA_PROG_ERROR: Invalid arguments, e.g. *strm doesn't - * support memory usage limit or memlimit was zero. + * support memory usage limit. */ extern LZMA_API(lzma_ret) lzma_memlimit_set( lzma_stream *strm, uint64_t memlimit) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/block.h b/Utilities/cmliblzma/liblzma/api/lzma/block.h index e6710a7..7bdcfd7 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/block.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/block.h @@ -31,11 +31,16 @@ typedef struct { /** * \brief Block format version * - * To prevent API and ABI breakages if new features are needed in - * the Block field, a version number is used to indicate which - * fields in this structure are in use. For now, version must always - * be zero. With non-zero version, most Block related functions will - * return LZMA_OPTIONS_ERROR. + * To prevent API and ABI breakages when new features are needed, + * a version number is used to indicate which fields in this + * structure are in use: + * - liblzma >= 5.0.0: version = 0 is supported. + * - liblzma >= 5.1.4beta: Support for version = 1 was added, + * which adds the ignore_check field. + * + * If version is greater than one, most Block related functions + * will return LZMA_OPTIONS_ERROR (lzma_block_header_decode() works + * with any version value). * * Read by: * - All functions that take pointer to lzma_block as argument, @@ -233,7 +238,28 @@ typedef struct { lzma_reserved_enum reserved_enum2; lzma_reserved_enum reserved_enum3; lzma_reserved_enum reserved_enum4; - lzma_bool reserved_bool1; + + /** + * \brief A flag to Block decoder to not verify the Check field + * + * This field is supported by liblzma >= 5.1.4beta if .version >= 1. + * + * If this is set to true, the integrity check won't be calculated + * and verified. Unless you know what you are doing, you should + * leave this to false. (A reason to set this to true is when the + * file integrity is verified externally anyway and you want to + * speed up the decompression, which matters mostly when using + * SHA-256 as the integrity check.) + * + * If .version >= 1, read by: + * - lzma_block_decoder() + * - lzma_block_buffer_decode() + * + * Written by (.version is ignored): + * - lzma_block_header_decode() always sets this to false + */ + lzma_bool ignore_check; + lzma_bool reserved_bool2; lzma_bool reserved_bool3; lzma_bool reserved_bool4; @@ -310,10 +336,14 @@ extern LZMA_API(lzma_ret) lzma_block_header_encode( /** * \brief Decode Block Header * - * block->version should be set to the highest value supported by the - * application; currently the only possible version is zero. This function - * will set version to the lowest value that still supports all the features - * required by the Block Header. + * block->version should (usually) be set to the highest value supported + * by the application. If the application sets block->version to a value + * higher than supported by the current liblzma version, this function will + * downgrade block->version to the highest value supported by it. Thus one + * should check the value of block->version after calling this function if + * block->version was set to a non-zero value and the application doesn't + * otherwise know that the liblzma version being used is new enough to + * support the specified block->version. * * The size of the Block Header must have already been decoded with * lzma_block_header_size_decode() macro and stored to block->header_size. @@ -344,7 +374,7 @@ extern LZMA_API(lzma_ret) lzma_block_header_encode( * block->header_size is invalid or block->filters is NULL. */ extern LZMA_API(lzma_ret) lzma_block_header_decode(lzma_block *block, - lzma_allocator *allocator, const uint8_t *in) + const lzma_allocator *allocator, const uint8_t *in) lzma_nothrow lzma_attr_warn_unused_result; @@ -493,7 +523,25 @@ extern LZMA_API(size_t) lzma_block_buffer_bound(size_t uncompressed_size) * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_block_buffer_encode( - lzma_block *block, lzma_allocator *allocator, + lzma_block *block, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** + * \brief Single-call uncompressed .xz Block encoder + * + * This is like lzma_block_buffer_encode() except this doesn't try to + * compress the data and instead encodes the data using LZMA2 uncompressed + * chunks. The required output buffer size can be determined with + * lzma_block_buffer_bound(). + * + * Since the data won't be compressed, this function ignores block->filters. + * This function doesn't take lzma_allocator because this function doesn't + * allocate any memory from the heap. + */ +extern LZMA_API(lzma_ret) lzma_block_uncomp_encode(lzma_block *block, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -527,7 +575,7 @@ extern LZMA_API(lzma_ret) lzma_block_buffer_encode( * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_block_buffer_decode( - lzma_block *block, lzma_allocator *allocator, + lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/container.h b/Utilities/cmliblzma/liblzma/api/lzma/container.h index 7a9ffc6..9fbf4df 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/container.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/container.h @@ -61,6 +61,131 @@ /** + * \brief Multithreading options + */ +typedef struct { + /** + * \brief Flags + * + * Set this to zero if no flags are wanted. + * + * No flags are currently supported. + */ + uint32_t flags; + + /** + * \brief Number of worker threads to use + */ + uint32_t threads; + + /** + * \brief Maximum uncompressed size of a Block + * + * The encoder will start a new .xz Block every block_size bytes. + * Using LZMA_FULL_FLUSH or LZMA_FULL_BARRIER with lzma_code() + * the caller may tell liblzma to start a new Block earlier. + * + * With LZMA2, a recommended block size is 2-4 times the LZMA2 + * dictionary size. With very small dictionaries, it is recommended + * to use at least 1 MiB block size for good compression ratio, even + * if this is more than four times the dictionary size. Note that + * these are only recommendations for typical use cases; feel free + * to use other values. Just keep in mind that using a block size + * less than the LZMA2 dictionary size is waste of RAM. + * + * Set this to 0 to let liblzma choose the block size depending + * on the compression options. For LZMA2 it will be 3*dict_size + * or 1 MiB, whichever is more. + * + * For each thread, about 3 * block_size bytes of memory will be + * allocated. This may change in later liblzma versions. If so, + * the memory usage will probably be reduced, not increased. + */ + uint64_t block_size; + + /** + * \brief Timeout to allow lzma_code() to return early + * + * Multithreading can make liblzma to consume input and produce + * output in a very bursty way: it may first read a lot of input + * to fill internal buffers, then no input or output occurs for + * a while. + * + * In single-threaded mode, lzma_code() won't return until it has + * either consumed all the input or filled the output buffer. If + * this is done in multithreaded mode, it may cause a call + * lzma_code() to take even tens of seconds, which isn't acceptable + * in all applications. + * + * To avoid very long blocking times in lzma_code(), a timeout + * (in milliseconds) may be set here. If lzma_code() would block + * longer than this number of milliseconds, it will return with + * LZMA_OK. Reasonable values are 100 ms or more. The xz command + * line tool uses 300 ms. + * + * If long blocking times are fine for you, set timeout to a special + * value of 0, which will disable the timeout mechanism and will make + * lzma_code() block until all the input is consumed or the output + * buffer has been filled. + * + * \note Even with a timeout, lzma_code() might sometimes take + * somewhat long time to return. No timing guarantees + * are made. + */ + uint32_t timeout; + + /** + * \brief Compression preset (level and possible flags) + * + * The preset is set just like with lzma_easy_encoder(). + * The preset is ignored if filters below is non-NULL. + */ + uint32_t preset; + + /** + * \brief Filter chain (alternative to a preset) + * + * If this is NULL, the preset above is used. Otherwise the preset + * is ignored and the filter chain specified here is used. + */ + const lzma_filter *filters; + + /** + * \brief Integrity check type + * + * See check.h for available checks. The xz command line tool + * defaults to LZMA_CHECK_CRC64, which is a good choice if you + * are unsure. + */ + lzma_check check; + + /* + * Reserved space to allow possible future extensions without + * breaking the ABI. You should not touch these, because the names + * of these variables may change. These are and will never be used + * with the currently supported options, so it is safe to leave these + * uninitialized. + */ + lzma_reserved_enum reserved_enum1; + lzma_reserved_enum reserved_enum2; + lzma_reserved_enum reserved_enum3; + uint32_t reserved_int1; + uint32_t reserved_int2; + uint32_t reserved_int3; + uint32_t reserved_int4; + uint64_t reserved_int5; + uint64_t reserved_int6; + uint64_t reserved_int7; + uint64_t reserved_int8; + void *reserved_ptr1; + void *reserved_ptr2; + void *reserved_ptr3; + void *reserved_ptr4; + +} lzma_mt; + + +/** * \brief Calculate approximate memory usage of easy encoder * * This function is a wrapper for lzma_raw_encoder_memusage(). @@ -165,7 +290,8 @@ extern LZMA_API(lzma_ret) lzma_easy_encoder( */ extern LZMA_API(lzma_ret) lzma_easy_buffer_encode( uint32_t preset, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -191,6 +317,49 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder(lzma_stream *strm, /** + * \brief Calculate approximate memory usage of multithreaded .xz encoder + * + * Since doing the encoding in threaded mode doesn't affect the memory + * requirements of single-threaded decompressor, you can use + * lzma_easy_decoder_memusage(options->preset) or + * lzma_raw_decoder_memusage(options->filters) to calculate + * the decompressor memory requirements. + * + * \param options Compression options + * + * \return Number of bytes of memory required for encoding with the + * given options. If an error occurs, for example due to + * unsupported preset or filter chain, UINT64_MAX is returned. + */ +extern LZMA_API(uint64_t) lzma_stream_encoder_mt_memusage( + const lzma_mt *options) lzma_nothrow lzma_attr_pure; + + +/** + * \brief Initialize multithreaded .xz Stream encoder + * + * This provides the functionality of lzma_easy_encoder() and + * lzma_stream_encoder() as a single function for multithreaded use. + * + * The supported actions for lzma_code() are LZMA_RUN, LZMA_FULL_FLUSH, + * LZMA_FULL_BARRIER, and LZMA_FINISH. Support for LZMA_SYNC_FLUSH might be + * added in the future. + * + * \param strm Pointer to properly prepared lzma_stream + * \param options Pointer to multithreaded compression options + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_UNSUPPORTED_CHECK + * - LZMA_OPTIONS_ERROR + * - LZMA_PROG_ERROR + */ +extern LZMA_API(lzma_ret) lzma_stream_encoder_mt( + lzma_stream *strm, const lzma_mt *options) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** * \brief Initialize .lzma encoder (legacy file format) * * The .lzma format is sometimes called the LZMA_Alone format, which is the @@ -269,7 +438,8 @@ extern LZMA_API(size_t) lzma_stream_buffer_bound(size_t uncompressed_size) */ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( lzma_filter *filters, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -305,6 +475,30 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( /** + * This flag makes lzma_code() not calculate and verify the integrity check + * of the compressed data in .xz files. This means that invalid integrity + * check values won't be detected and LZMA_DATA_ERROR won't be returned in + * such cases. + * + * This flag only affects the checks of the compressed data itself; the CRC32 + * values in the .xz headers will still be verified normally. + * + * Don't use this flag unless you know what you are doing. Possible reasons + * to use this flag: + * + * - Trying to recover data from a corrupt .xz file. + * + * - Speeding up decompression, which matters mostly with SHA-256 + * or with files that have compressed extremely well. It's recommended + * to not use this flag for this purpose unless the file integrity is + * verified externally in some other way. + * + * Support for this flag was added in liblzma 5.1.4beta. + */ +#define LZMA_IGNORE_CHECK UINT32_C(0x10) + + +/** * This flag enables decoding of concatenated files with file formats that * allow concatenating compressed files as is. From the formats currently * supported by liblzma, only the .xz format allows concatenated files. @@ -326,7 +520,10 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( * * \param strm Pointer to properly prepared lzma_stream * \param memlimit Memory usage limit as bytes. Use UINT64_MAX - * to effectively disable the limiter. + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. * \param flags Bitwise-or of zero or more of the decoder flags: * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, * LZMA_TELL_ANY_CHECK, LZMA_CONCATENATED @@ -350,7 +547,10 @@ extern LZMA_API(lzma_ret) lzma_stream_decoder( * * \param strm Pointer to properly prepared lzma_stream * \param memlimit Memory usage limit as bytes. Use UINT64_MAX - * to effectively disable the limiter. + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. * \param flags Bitwise-or of flags, or zero for no flags. * * \return - LZMA_OK: Initialization was successful. @@ -366,9 +566,16 @@ extern LZMA_API(lzma_ret) lzma_auto_decoder( /** * \brief Initialize .lzma decoder (legacy file format) * + * \param strm Pointer to properly prepared lzma_stream + * \param memlimit Memory usage limit as bytes. Use UINT64_MAX + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. + * * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. - * There is no need to use LZMA_FINISH, but allowing it may simplify - * certain types of applications. + * There is no need to use LZMA_FINISH, but it's allowed because it may + * simplify certain types of applications. * * \return - LZMA_OK * - LZMA_MEM_ERROR @@ -418,7 +625,8 @@ extern LZMA_API(lzma_ret) lzma_alone_decoder( * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_buffer_decode( - uint64_t *memlimit, uint32_t flags, lzma_allocator *allocator, + uint64_t *memlimit, uint32_t flags, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/filter.h b/Utilities/cmliblzma/liblzma/api/lzma/filter.h index e0bc163..4e78752 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/filter.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/filter.h @@ -116,8 +116,9 @@ extern LZMA_API(lzma_bool) lzma_filter_decoder_is_supported(lzma_vli id) * is not NULL. * - LZMA_PROG_ERROR: src or dest is NULL. */ -extern LZMA_API(lzma_ret) lzma_filters_copy(const lzma_filter *src, - lzma_filter *dest, lzma_allocator *allocator) lzma_nothrow; +extern LZMA_API(lzma_ret) lzma_filters_copy( + const lzma_filter *src, lzma_filter *dest, + const lzma_allocator *allocator) lzma_nothrow; /** @@ -256,7 +257,7 @@ extern LZMA_API(lzma_ret) lzma_filters_update( * won't necessarily meet that bound.) */ extern LZMA_API(lzma_ret) lzma_raw_buffer_encode( - const lzma_filter *filters, lzma_allocator *allocator, + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -280,7 +281,7 @@ extern LZMA_API(lzma_ret) lzma_raw_buffer_encode( * which no data is written to is out[out_size]. */ extern LZMA_API(lzma_ret) lzma_raw_buffer_decode( - const lzma_filter *filters, lzma_allocator *allocator, + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -356,7 +357,7 @@ extern LZMA_API(lzma_ret) lzma_properties_encode( * - LZMA_MEM_ERROR */ extern LZMA_API(lzma_ret) lzma_properties_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) lzma_nothrow; @@ -419,6 +420,6 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filter, * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_filter_flags_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) lzma_nothrow lzma_attr_warn_unused_result; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/hardware.h b/Utilities/cmliblzma/liblzma/api/lzma/hardware.h index e7dd03c..5321d9a 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/hardware.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/hardware.h @@ -48,3 +48,17 @@ * of RAM on the specific operating system. */ extern LZMA_API(uint64_t) lzma_physmem(void) lzma_nothrow; + + +/** + * \brief Get the number of processor cores or threads + * + * This function may be useful when determining how many threads to use. + * If the hardware supports more than one thread per CPU core, the number + * of hardware threads is returned if that information is available. + * + * \brief On success, the number of available CPU threads or cores is + * returned. If this information isn't available or an error + * occurs, zero is returned. + */ +extern LZMA_API(uint32_t) lzma_cputhreads(void) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/index.h b/Utilities/cmliblzma/liblzma/api/lzma/index.h index 16bacc2..3dac6fb 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/index.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/index.h @@ -303,7 +303,7 @@ extern LZMA_API(uint64_t) lzma_index_memused(const lzma_index *i) * \return On success, a pointer to an empty initialized lzma_index is * returned. If allocation fails, NULL is returned. */ -extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator) +extern LZMA_API(lzma_index *) lzma_index_init(const lzma_allocator *allocator) lzma_nothrow; @@ -312,8 +312,8 @@ extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator) * * If i is NULL, this does nothing. */ -extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator) - lzma_nothrow; +extern LZMA_API(void) lzma_index_end( + lzma_index *i, const lzma_allocator *allocator) lzma_nothrow; /** @@ -341,7 +341,7 @@ extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator) * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_index_append( - lzma_index *i, lzma_allocator *allocator, + lzma_index *i, const lzma_allocator *allocator, lzma_vli unpadded_size, lzma_vli uncompressed_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -564,8 +564,8 @@ extern LZMA_API(lzma_bool) lzma_index_iter_locate( * - LZMA_MEM_ERROR * - LZMA_PROG_ERROR */ -extern LZMA_API(lzma_ret) lzma_index_cat( - lzma_index *dest, lzma_index *src, lzma_allocator *allocator) +extern LZMA_API(lzma_ret) lzma_index_cat(lzma_index *dest, lzma_index *src, + const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -575,7 +575,7 @@ extern LZMA_API(lzma_ret) lzma_index_cat( * \return A copy of the lzma_index, or NULL if memory allocation failed. */ extern LZMA_API(lzma_index *) lzma_index_dup( - const lzma_index *i, lzma_allocator *allocator) + const lzma_index *i, const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -586,8 +586,7 @@ extern LZMA_API(lzma_index *) lzma_index_dup( * \param i Pointer to lzma_index which should be encoded. * * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. - * It is enough to use only one of them (you can choose freely; use LZMA_RUN - * to support liblzma versions older than 5.0.0). + * It is enough to use only one of them (you can choose freely). * * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR @@ -610,16 +609,21 @@ extern LZMA_API(lzma_ret) lzma_index_encoder( * to a new lzma_index, which the application * has to later free with lzma_index_end(). * \param memlimit How much memory the resulting lzma_index is - * allowed to require. + * allowed to require. liblzma 5.2.3 and earlier + * don't allow 0 here and return LZMA_PROG_ERROR; + * later versions treat 0 as if 1 had been specified. * - * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. - * It is enough to use only one of them (you can choose freely; use LZMA_RUN - * to support liblzma versions older than 5.0.0). + * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * There is no need to use LZMA_FINISH, but it's allowed because it may + * simplify certain types of applications. * * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR - * - LZMA_MEMLIMIT_ERROR * - LZMA_PROG_ERROR + * + * liblzma 5.2.3 and older list also LZMA_MEMLIMIT_ERROR here + * but that error code has never been possible from this + * initialization function. */ extern LZMA_API(lzma_ret) lzma_index_decoder( lzma_stream *strm, lzma_index **i, uint64_t memlimit) @@ -677,6 +681,6 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i, * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_index_buffer_decode(lzma_index **i, - uint64_t *memlimit, lzma_allocator *allocator, + uint64_t *memlimit, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h b/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h index fa2e048..9287f1d 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h @@ -37,7 +37,7 @@ typedef struct lzma_index_hash_s lzma_index_hash; * pointer than the index_hash that was given as an argument. */ extern LZMA_API(lzma_index_hash *) lzma_index_hash_init( - lzma_index_hash *index_hash, lzma_allocator *allocator) + lzma_index_hash *index_hash, const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -45,7 +45,7 @@ extern LZMA_API(lzma_index_hash *) lzma_index_hash_init( * \brief Deallocate lzma_index_hash structure */ extern LZMA_API(void) lzma_index_hash_end( - lzma_index_hash *index_hash, lzma_allocator *allocator) + lzma_index_hash *index_hash, const lzma_allocator *allocator) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/lzma.h b/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h index 3f8e095..4e32fa3 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/lzma.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h @@ -1,5 +1,5 @@ /** - * \file lzma/lzma.h + * \file lzma/lzma12.h * \brief LZMA1 and LZMA2 filters */ diff --git a/Utilities/cmliblzma/liblzma/api/lzma/version.h b/Utilities/cmliblzma/liblzma/api/lzma/version.h index 09866b9..143c7de 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/version.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/version.h @@ -21,8 +21,8 @@ * Version number split into components */ #define LZMA_VERSION_MAJOR 5 -#define LZMA_VERSION_MINOR 0 -#define LZMA_VERSION_PATCH 8 +#define LZMA_VERSION_MINOR 2 +#define LZMA_VERSION_PATCH 4 #define LZMA_VERSION_STABILITY LZMA_VERSION_STABILITY_STABLE #ifndef LZMA_VERSION_COMMIT diff --git a/Utilities/cmliblzma/liblzma/check/check.c b/Utilities/cmliblzma/liblzma/check/check.c index 979b0a8..428ddae 100644 --- a/Utilities/cmliblzma/liblzma/check/check.c +++ b/Utilities/cmliblzma/liblzma/check/check.c @@ -16,6 +16,9 @@ extern LZMA_API(lzma_bool) lzma_check_is_supported(lzma_check type) { + if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) + return false; + static const lzma_bool available_checks[LZMA_CHECK_ID_MAX + 1] = { true, // LZMA_CHECK_NONE @@ -53,9 +56,6 @@ lzma_check_is_supported(lzma_check type) false, // Reserved }; - if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) - return false; - return available_checks[(unsigned int)(type)]; } @@ -63,6 +63,9 @@ lzma_check_is_supported(lzma_check type) extern LZMA_API(uint32_t) lzma_check_size(lzma_check type) { + if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) + return UINT32_MAX; + // See file-format.txt section 2.1.1.2. static const uint8_t check_sizes[LZMA_CHECK_ID_MAX + 1] = { 0, @@ -73,9 +76,6 @@ lzma_check_size(lzma_check type) 64, 64, 64 }; - if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) - return UINT32_MAX; - return check_sizes[(unsigned int)(type)]; } diff --git a/Utilities/cmliblzma/liblzma/check/check.h b/Utilities/cmliblzma/liblzma/check/check.h index e100d2b..3007d88 100644 --- a/Utilities/cmliblzma/liblzma/check/check.h +++ b/Utilities/cmliblzma/liblzma/check/check.h @@ -15,6 +15,53 @@ #include "common.h" +// If the function for external SHA-256 is missing, use the internal SHA-256 +// code. Due to how configure works, these defines can only get defined when +// both a usable header and a type have already been found. +#if !(defined(HAVE_CC_SHA256_INIT) \ + || defined(HAVE_SHA256_INIT) \ + || defined(HAVE_SHA256INIT)) +# define HAVE_INTERNAL_SHA256 1 +#endif + +#if defined(HAVE_INTERNAL_SHA256) +// Nothing +#elif defined(HAVE_COMMONCRYPTO_COMMONDIGEST_H) +# include <CommonCrypto/CommonDigest.h> +#elif defined(HAVE_SHA256_H) +# include <sys/types.h> +# include <sha256.h> +#elif defined(HAVE_SHA2_H) +# include <sys/types.h> +# include <sha2.h> +#endif + +#if defined(HAVE_INTERNAL_SHA256) +/// State for the internal SHA-256 implementation +typedef struct { + /// Internal state + uint32_t state[8]; + + /// Size of the message excluding padding + uint64_t size; +} lzma_sha256_state; +#elif defined(HAVE_CC_SHA256_CTX) +typedef CC_SHA256_CTX lzma_sha256_state; +#elif defined(HAVE_SHA256_CTX) +typedef SHA256_CTX lzma_sha256_state; +#elif defined(HAVE_SHA2_CTX) +typedef SHA2_CTX lzma_sha256_state; +#endif + +#if defined(HAVE_INTERNAL_SHA256) +// Nothing +#elif defined(HAVE_CC_SHA256_INIT) +# define LZMA_SHA256FUNC(x) CC_SHA256_ ## x +#elif defined(HAVE_SHA256_INIT) +# define LZMA_SHA256FUNC(x) SHA256_ ## x +#elif defined(HAVE_SHA256INIT) +# define LZMA_SHA256FUNC(x) SHA256 ## x +#endif // Index hashing needs the best possible hash function (preferably // a cryptographic hash) for maximum reliability. @@ -43,14 +90,7 @@ typedef struct { union { uint32_t crc32; uint64_t crc64; - - struct { - /// Internal state - uint32_t state[8]; - - /// Size of the message excluding padding - uint64_t size; - } sha256; + lzma_sha256_state sha256; } state; } lzma_check_state; @@ -82,6 +122,8 @@ extern void lzma_check_update(lzma_check_state *check, lzma_check type, extern void lzma_check_finish(lzma_check_state *check, lzma_check type); +#ifndef LZMA_SHA256FUNC + /// Prepare SHA-256 state for new input. extern void lzma_sha256_init(lzma_check_state *check); @@ -92,4 +134,39 @@ extern void lzma_sha256_update( /// Finish the SHA-256 calculation and store the result to check->buffer.u8. extern void lzma_sha256_finish(lzma_check_state *check); + +#else + +static inline void +lzma_sha256_init(lzma_check_state *check) +{ + LZMA_SHA256FUNC(Init)(&check->state.sha256); +} + + +static inline void +lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) +{ +#if defined(HAVE_CC_SHA256_INIT) && SIZE_MAX > UINT32_MAX + // Darwin's CC_SHA256_Update takes uint32_t as the buffer size, + // so use a loop to support size_t. + while (size > UINT32_MAX) { + LZMA_SHA256FUNC(Update)(&check->state.sha256, buf, UINT32_MAX); + buf += UINT32_MAX; + size -= UINT32_MAX; + } +#endif + + LZMA_SHA256FUNC(Update)(&check->state.sha256, buf, size); +} + + +static inline void +lzma_sha256_finish(lzma_check_state *check) +{ + LZMA_SHA256FUNC(Final)(check->buffer.u8, &check->state.sha256); +} + +#endif + #endif diff --git a/Utilities/cmliblzma/liblzma/check/crc32_fast.c b/Utilities/cmliblzma/liblzma/check/crc32_fast.c index c2c3cb7..3de0263 100644 --- a/Utilities/cmliblzma/liblzma/check/crc32_fast.c +++ b/Utilities/cmliblzma/liblzma/check/crc32_fast.c @@ -33,8 +33,6 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) #endif if (size > 8) { - const uint8_t * limit; - // Fix the alignment, if needed. The if statement above // ensures that this won't read past the end of buf[]. while ((uintptr_t)(buf) & 7) { @@ -43,7 +41,7 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) } // Calculate the position where to stop. - limit = buf + (size & ~(size_t)(7)); + const uint8_t *const limit = buf + (size & ~(size_t)(7)); // Calculate how many bytes must be calculated separately // before returning the result. @@ -51,8 +49,6 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) // Calculate the CRC32 using the slice-by-eight algorithm. while (buf < limit) { - uint32_t tmp; - crc ^= *(const uint32_t *)(buf); buf += 4; @@ -61,7 +57,7 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) ^ lzma_crc32_table[5][C(crc)] ^ lzma_crc32_table[4][D(crc)]; - tmp = *(const uint32_t *)(buf); + const uint32_t tmp = *(const uint32_t *)(buf); buf += 4; // At least with some compilers, it is critical for diff --git a/Utilities/cmliblzma/liblzma/check/crc64_fast.c b/Utilities/cmliblzma/liblzma/check/crc64_fast.c index 1436557..52af29e 100644 --- a/Utilities/cmliblzma/liblzma/check/crc64_fast.c +++ b/Utilities/cmliblzma/liblzma/check/crc64_fast.c @@ -36,14 +36,12 @@ lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) #endif if (size > 4) { - const uint8_t *limit; - while ((uintptr_t)(buf) & 3) { crc = lzma_crc64_table[0][*buf++ ^ A1(crc)] ^ S8(crc); --size; } - limit = buf + (size & ~(size_t)(3)); + const uint8_t *const limit = buf + (size & ~(size_t)(3)); size &= (size_t)(3); while (buf < limit) { diff --git a/Utilities/cmliblzma/liblzma/check/sha256.c b/Utilities/cmliblzma/liblzma/check/sha256.c index 3af6aa6..5eede5c 100644 --- a/Utilities/cmliblzma/liblzma/check/sha256.c +++ b/Utilities/cmliblzma/liblzma/check/sha256.c @@ -21,22 +21,22 @@ // /////////////////////////////////////////////////////////////////////////////// -// Avoid bogus warnings in transform(). -#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 2) || __GNUC__ > 4 -# pragma GCC diagnostic ignored "-Wuninitialized" -#endif - #include "check.h" -// At least on x86, GCC is able to optimize this to a rotate instruction. -#define rotr_32(num, amount) ((num) >> (amount) | (num) << (32 - (amount))) +// Rotate a uint32_t. GCC can optimize this to a rotate instruction +// at least on x86. +static inline uint32_t +rotr_32(uint32_t num, unsigned amount) +{ + return (num >> amount) | (num << (32 - amount)); +} -#define blk0(i) (W[i] = data[i]) +#define blk0(i) (W[i] = conv32be(data[i])) #define blk2(i) (W[i & 15] += s1(W[(i - 2) & 15]) + W[(i - 7) & 15] \ + s0(W[(i - 15) & 15])) #define Ch(x, y, z) (z ^ (x & (y ^ z))) -#define Maj(x, y, z) ((x & y) | (z & (x | y))) +#define Maj(x, y, z) ((x & (y ^ z)) + (y & z)) #define a(i) T[(0 - i) & 7] #define b(i) T[(1 - i) & 7] @@ -47,16 +47,17 @@ #define g(i) T[(6 - i) & 7] #define h(i) T[(7 - i) & 7] -#define R(i) \ - h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] \ - + (j ? blk2(i) : blk0(i)); \ +#define R(i, j, blk) \ + h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] + blk; \ d(i) += h(i); \ h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) +#define R0(i) R(i, 0, blk0(i)) +#define R2(i) R(i, j, blk2(i)) -#define S0(x) (rotr_32(x, 2) ^ rotr_32(x, 13) ^ rotr_32(x, 22)) -#define S1(x) (rotr_32(x, 6) ^ rotr_32(x, 11) ^ rotr_32(x, 25)) -#define s0(x) (rotr_32(x, 7) ^ rotr_32(x, 18) ^ (x >> 3)) -#define s1(x) (rotr_32(x, 17) ^ rotr_32(x, 19) ^ (x >> 10)) +#define S0(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 9), 11), 2) +#define S1(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 14), 5), 6) +#define s0(x) (rotr_32(x ^ rotr_32(x, 11), 7) ^ (x >> 3)) +#define s1(x) (rotr_32(x ^ rotr_32(x, 2), 17) ^ (x >> 10)) static const uint32_t SHA256_K[64] = { @@ -84,17 +85,22 @@ transform(uint32_t state[8], const uint32_t data[16]) { uint32_t W[16]; uint32_t T[8]; - unsigned int j; // Copy state[] to working vars. memcpy(T, state, sizeof(T)); - // 64 operations, partially loop unrolled - for (j = 0; j < 64; j += 16) { - R( 0); R( 1); R( 2); R( 3); - R( 4); R( 5); R( 6); R( 7); - R( 8); R( 9); R(10); R(11); - R(12); R(13); R(14); R(15); + // The first 16 operations unrolled + R0( 0); R0( 1); R0( 2); R0( 3); + R0( 4); R0( 5); R0( 6); R0( 7); + R0( 8); R0( 9); R0(10); R0(11); + R0(12); R0(13); R0(14); R0(15); + + // The remaining 48 operations partially unrolled + for (unsigned int j = 16; j < 64; j += 16) { + R2( 0); R2( 1); R2( 2); R2( 3); + R2( 4); R2( 5); R2( 6); R2( 7); + R2( 8); R2( 9); R2(10); R2(11); + R2(12); R2(13); R2(14); R2(15); } // Add the working vars back into state[]. @@ -112,19 +118,7 @@ transform(uint32_t state[8], const uint32_t data[16]) static void process(lzma_check_state *check) { -#ifdef WORDS_BIGENDIAN transform(check->state.sha256.state, check->buffer.u32); - -#else - uint32_t data[16]; - size_t i; - - for (i = 0; i < 16; ++i) - data[i] = bswap32(check->buffer.u32[i]); - - transform(check->state.sha256.state, data); -#endif - return; } @@ -174,8 +168,6 @@ lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) extern void lzma_sha256_finish(lzma_check_state *check) { - size_t i; - // Add padding as described in RFC 3174 (it describes SHA-1 but // the same padding style is used for SHA-256 too). size_t pos = check->state.sha256.size & 0x3F; @@ -197,7 +189,7 @@ lzma_sha256_finish(lzma_check_state *check) process(check); - for (i = 0; i < 8; ++i) + for (size_t i = 0; i < 8; ++i) check->buffer.u32[i] = conv32be(check->state.sha256.state[i]); return; diff --git a/Utilities/cmliblzma/liblzma/common/alone_decoder.c b/Utilities/cmliblzma/liblzma/common/alone_decoder.c index 5f5e564..77d0a9b 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/alone_decoder.c @@ -15,7 +15,7 @@ #include "lz_decoder.h" -struct lzma_coder_s { +typedef struct { lzma_next_coder next; enum { @@ -46,17 +46,19 @@ struct lzma_coder_s { /// Options decoded from the header needed to initialize /// the LZMA decoder lzma_options_lzma options; -}; +} lzma_alone_coder; static lzma_ret -alone_decode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +alone_decode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_alone_coder *coder = coder_ptr; + while (*out_pos < out_size && (coder->sequence == SEQ_CODE || *in_pos < in_size)) switch (coder->sequence) { @@ -126,17 +128,19 @@ alone_decode(lzma_coder *coder, // Fall through case SEQ_CODER_INIT: { - lzma_ret ret; + if (coder->memusage > coder->memlimit) + return LZMA_MEMLIMIT_ERROR; lzma_filter_info filters[2] = { - { 0, &lzma_lzma_decoder_init, &coder->options }, - { 0, NULL, NULL } + { + .init = &lzma_lzma_decoder_init, + .options = &coder->options, + }, { + .init = NULL, + } }; - if (coder->memusage > coder->memlimit) - return LZMA_MEMLIMIT_ERROR; - - ret = lzma_next_filter_init(&coder->next, + const lzma_ret ret = lzma_next_filter_init(&coder->next, allocator, filters); if (ret != LZMA_OK) return ret; @@ -164,8 +168,9 @@ alone_decode(lzma_coder *coder, static void -alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +alone_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_alone_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -173,9 +178,11 @@ alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -alone_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +alone_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_alone_coder *coder = coder_ptr; + *memusage = coder->memusage; *old_memlimit = coder->memlimit; @@ -191,34 +198,34 @@ alone_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, extern lzma_ret -lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_alone_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, bool picky) { lzma_next_coder_init(&lzma_alone_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; + lzma_alone_coder *coder = next->coder; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_alone_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &alone_decode; next->end = &alone_decoder_end; next->memconfig = &alone_decoder_memconfig; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } - next->coder->sequence = SEQ_PROPERTIES; - next->coder->picky = picky; - next->coder->pos = 0; - next->coder->options.dict_size = 0; - next->coder->options.preset_dict = NULL; - next->coder->options.preset_dict_size = 0; - next->coder->uncompressed_size = 0; - next->coder->memlimit = memlimit; - next->coder->memusage = LZMA_MEMUSAGE_BASE; + coder->sequence = SEQ_PROPERTIES; + coder->picky = picky; + coder->pos = 0; + coder->options.dict_size = 0; + coder->options.preset_dict = NULL; + coder->options.preset_dict_size = 0; + coder->uncompressed_size = 0; + coder->memlimit = my_max(1, memlimit); + coder->memusage = LZMA_MEMUSAGE_BASE; return LZMA_OK; } @@ -227,7 +234,7 @@ lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit) { - lzma_next_strm_init2(lzma_alone_decoder_init, strm, memlimit, false); + lzma_next_strm_init(lzma_alone_decoder_init, strm, memlimit, false); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/alone_decoder.h b/Utilities/cmliblzma/liblzma/common/alone_decoder.h index f666fc3..dfa031a 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/alone_decoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_alone_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, bool picky); #endif diff --git a/Utilities/cmliblzma/liblzma/common/alone_encoder.c b/Utilities/cmliblzma/liblzma/common/alone_encoder.c index 4207b4a..4853cfd 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/alone_encoder.c @@ -17,7 +17,7 @@ #define ALONE_HEADER_SIZE (1 + 4 + 8) -struct lzma_coder_s { +typedef struct { lzma_next_coder next; enum { @@ -27,17 +27,19 @@ struct lzma_coder_s { size_t header_pos; uint8_t header[ALONE_HEADER_SIZE]; -}; +} lzma_alone_coder; static lzma_ret -alone_encode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +alone_encode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_alone_coder *coder = coder_ptr; + while (*out_pos < out_size) switch (coder->sequence) { case SEQ_HEADER: @@ -65,8 +67,9 @@ alone_encode(lzma_coder *coder, static void -alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +alone_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_alone_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -75,36 +78,31 @@ alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) // At least for now, this is not used by any internal function. static lzma_ret -alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_options_lzma *options) { - uint32_t d; - - // Initialize the LZMA encoder. - const lzma_filter_info filters[2] = { - { 0, &lzma_lzma_encoder_init, (void *)(options) }, - { 0, NULL, NULL } - }; - lzma_next_coder_init(&alone_encoder_init, next, allocator); - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_alone_coder *coder = next->coder; + + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_alone_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &alone_encode; next->end = &alone_encoder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_HEADER; - next->coder->header_pos = 0; + coder->sequence = SEQ_HEADER; + coder->header_pos = 0; // Encode the header: // - Properties (1 byte) - if (lzma_lzma_lclppb_encode(options, next->coder->header)) + if (lzma_lzma_lclppb_encode(options, coder->header)) return LZMA_OPTIONS_ERROR; // - Dictionary size (4 bytes) @@ -115,7 +113,7 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // one is the next unless it is UINT32_MAX. While the header would // allow any 32-bit integer, we do this to keep the decoder of liblzma // accepting the resulting files. - d = options->dict_size - 1; + uint32_t d = options->dict_size - 1; d |= d >> 2; d |= d >> 3; d |= d >> 4; @@ -124,18 +122,28 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, if (d != UINT32_MAX) ++d; - unaligned_write32le(next->coder->header + 1, d); + unaligned_write32le(coder->header + 1, d); // - Uncompressed size (always unknown and using EOPM) - memset(next->coder->header + 1 + 4, 0xFF, 8); + memset(coder->header + 1 + 4, 0xFF, 8); + + // Initialize the LZMA encoder. + const lzma_filter_info filters[2] = { + { + .init = &lzma_lzma_encoder_init, + .options = (void *)(options), + }, { + .init = NULL, + } + }; - return lzma_next_filter_init(&next->coder->next, allocator, filters); + return lzma_next_filter_init(&coder->next, allocator, filters); } /* extern lzma_ret -lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_options_alone *options) { lzma_next_coder_init(&alone_encoder_init, next, allocator, options); @@ -146,7 +154,7 @@ lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_alone_encoder(lzma_stream *strm, const lzma_options_lzma *options) { - lzma_next_strm_init1(alone_encoder_init, strm, options); + lzma_next_strm_init(alone_encoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/auto_decoder.c b/Utilities/cmliblzma/liblzma/common/auto_decoder.c index 24cf489..6895c7c 100644 --- a/Utilities/cmliblzma/liblzma/common/auto_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/auto_decoder.c @@ -14,7 +14,7 @@ #include "alone_decoder.h" -struct lzma_coder_s { +typedef struct { /// Stream decoder or LZMA_Alone decoder lzma_next_coder next; @@ -26,15 +26,17 @@ struct lzma_coder_s { SEQ_CODE, SEQ_FINISH, } sequence; -}; +} lzma_auto_coder; static lzma_ret -auto_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +auto_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_auto_coder *coder = coder_ptr; + switch (coder->sequence) { case SEQ_INIT: if (*in_pos >= in_size) @@ -100,8 +102,9 @@ auto_decode(lzma_coder *coder, lzma_allocator *allocator, static void -auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +auto_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_auto_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -109,8 +112,10 @@ auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_check -auto_decoder_get_check(const lzma_coder *coder) +auto_decoder_get_check(const void *coder_ptr) { + const lzma_auto_coder *coder = coder_ptr; + // It is LZMA_Alone if get_check is NULL. return coder->next.get_check == NULL ? LZMA_CHECK_NONE : coder->next.get_check(coder->next.coder); @@ -118,9 +123,11 @@ auto_decoder_get_check(const lzma_coder *coder) static lzma_ret -auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +auto_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_auto_coder *coder = coder_ptr; + lzma_ret ret; if (coder->next.memconfig != NULL) { @@ -132,7 +139,10 @@ auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, // the current memory usage. *memusage = LZMA_MEMUSAGE_BASE; *old_memlimit = coder->memlimit; + ret = LZMA_OK; + if (new_memlimit != 0 && new_memlimit < *memusage) + ret = LZMA_MEMLIMIT_ERROR; } if (ret == LZMA_OK && new_memlimit != 0) @@ -143,32 +153,31 @@ auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, static lzma_ret -auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +auto_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, uint32_t flags) { lzma_next_coder_init(&auto_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; - if (flags & ~LZMA_SUPPORTED_FLAGS) return LZMA_OPTIONS_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_auto_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_auto_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &auto_decode; next->end = &auto_decoder_end; next->get_check = &auto_decoder_get_check; next->memconfig = &auto_decoder_memconfig; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } - next->coder->memlimit = memlimit; - next->coder->flags = flags; - next->coder->sequence = SEQ_INIT; + coder->memlimit = my_max(1, memlimit); + coder->flags = flags; + coder->sequence = SEQ_INIT; return LZMA_OK; } @@ -177,7 +186,7 @@ auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_auto_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init2(auto_decoder_init, strm, memlimit, flags); + lzma_next_strm_init(auto_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c index b4bd388..b0ded90 100644 --- a/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c @@ -14,13 +14,10 @@ extern LZMA_API(lzma_ret) -lzma_block_buffer_decode(lzma_block *block, lzma_allocator *allocator, +lzma_block_buffer_decode(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder block_decoder; - lzma_ret ret; - if (in_pos == NULL || (in == NULL && *in_pos != in_size) || *in_pos > in_size || out_pos == NULL || (out == NULL && *out_pos != out_size) @@ -28,8 +25,9 @@ lzma_block_buffer_decode(lzma_block *block, lzma_allocator *allocator, return LZMA_PROG_ERROR; // Initialize the Block decoder. - block_decoder = LZMA_NEXT_CODER_INIT; - ret = lzma_block_decoder_init(&block_decoder, allocator, block); + lzma_next_coder block_decoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_block_decoder_init( + &block_decoder, allocator, block); if (ret == LZMA_OK) { // Save the positions so that we can restore them in case diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c index 136f7f5..39e263a 100644 --- a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c @@ -10,6 +10,7 @@ // /////////////////////////////////////////////////////////////////////////////// +#include "block_buffer_encoder.h" #include "block_encoder.h" #include "filter_encoder.h" #include "lzma2_encoder.h" @@ -28,11 +29,9 @@ + LZMA_CHECK_SIZE_MAX + 3) & ~3) -static lzma_vli -lzma2_bound(lzma_vli uncompressed_size) +static uint64_t +lzma2_bound(uint64_t uncompressed_size) { - lzma_vli overhead; - // Prevent integer overflow in overhead calculation. if (uncompressed_size > COMPRESSED_SIZE_MAX) return 0; @@ -41,7 +40,7 @@ lzma2_bound(lzma_vli uncompressed_size) // uncompressed_size up to the next multiple of LZMA2_CHUNK_MAX, // multiply by the size of per-chunk header, and add one byte for // the end marker. - overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1) + const uint64_t overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1) / LZMA2_CHUNK_MAX) * LZMA2_HEADER_UNCOMPRESSED + 1; @@ -53,30 +52,36 @@ lzma2_bound(lzma_vli uncompressed_size) } -extern LZMA_API(size_t) -lzma_block_buffer_bound(size_t uncompressed_size) +extern uint64_t +lzma_block_buffer_bound64(uint64_t uncompressed_size) { - // For now, if the data doesn't compress, we always use uncompressed - // chunks of LZMA2. In future we may use Subblock filter too, but - // but for simplicity we probably will still use the same bound - // calculation even though Subblock filter would have slightly less - // overhead. - lzma_vli lzma2_size = lzma2_bound(uncompressed_size); + // If the data doesn't compress, we always use uncompressed + // LZMA2 chunks. + uint64_t lzma2_size = lzma2_bound(uncompressed_size); if (lzma2_size == 0) return 0; // Take Block Padding into account. - lzma2_size = (lzma2_size + 3) & ~LZMA_VLI_C(3); + lzma2_size = (lzma2_size + 3) & ~UINT64_C(3); -#if SIZE_MAX < LZMA_VLI_MAX - // Catch the possible integer overflow on 32-bit systems. There's no - // overflow on 64-bit systems, because lzma2_bound() already takes + // No risk of integer overflow because lzma2_bound() already takes // into account the size of the headers in the Block. - if (SIZE_MAX - HEADERS_BOUND < lzma2_size) + return HEADERS_BOUND + lzma2_size; +} + + +extern LZMA_API(size_t) +lzma_block_buffer_bound(size_t uncompressed_size) +{ + uint64_t ret = lzma_block_buffer_bound64(uncompressed_size); + +#if SIZE_MAX < UINT64_MAX + // Catch the possible integer overflow on 32-bit systems. + if (ret > SIZE_MAX) return 0; #endif - return HEADERS_BOUND + lzma2_size; + return ret; } @@ -84,17 +89,12 @@ static lzma_ret block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - size_t in_pos = 0; - uint8_t control = 0x01; // Dictionary reset - lzma_filter *filters_orig; - - // TODO: Figure out if the last filter is LZMA2 or Subblock and use - // that filter to encode the uncompressed chunks. - // Use LZMA2 uncompressed chunks. We wouldn't need a dictionary at // all, but LZMA2 always requires a dictionary, so use the minimum // value to minimize memory usage of the decoder. - lzma_options_lzma lzma2 = { LZMA_DICT_SIZE_MIN }; + lzma_options_lzma lzma2 = { + .dict_size = LZMA_DICT_SIZE_MIN, + }; lzma_filter filters[2]; filters[0].id = LZMA_FILTER_LZMA2; @@ -103,7 +103,7 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, // Set the above filter options to *block temporarily so that we can // encode the Block Header. - filters_orig = block->filters; + lzma_filter *filters_orig = block->filters; block->filters = filters; if (lzma_block_header_size(block) != LZMA_OK) { @@ -132,17 +132,18 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, *out_pos += block->header_size; // Encode the data using LZMA2 uncompressed chunks. + size_t in_pos = 0; + uint8_t control = 0x01; // Dictionary reset while (in_pos < in_size) { - size_t copy_size; - // Control byte: Indicate uncompressed chunk, of which // the first resets the dictionary. out[(*out_pos)++] = control; control = 0x02; // No dictionary reset // Size of the uncompressed chunk - copy_size = my_min(in_size - in_pos, LZMA2_CHUNK_MAX); + const size_t copy_size + = my_min(in_size - in_pos, LZMA2_CHUNK_MAX); out[(*out_pos)++] = (copy_size - 1) >> 8; out[(*out_pos)++] = (copy_size - 1) & 0xFF; @@ -163,27 +164,18 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, static lzma_ret -block_encode_normal(lzma_block *block, lzma_allocator *allocator, +block_encode_normal(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - size_t out_start; - lzma_next_coder raw_encoder = LZMA_NEXT_CODER_INIT; - lzma_ret ret; - // Find out the size of the Block Header. - block->compressed_size = lzma2_bound(in_size); - if (block->compressed_size == 0) - return LZMA_DATA_ERROR; - - block->uncompressed_size = in_size; return_if_error(lzma_block_header_size(block)); // Reserve space for the Block Header and skip it for now. if (out_size - *out_pos <= block->header_size) return LZMA_BUF_ERROR; - out_start = *out_pos; + const size_t out_start = *out_pos; *out_pos += block->header_size; // Limit out_size so that we stop encoding if the output would grow @@ -193,7 +185,8 @@ block_encode_normal(lzma_block *block, lzma_allocator *allocator, // TODO: In many common cases this could be optimized to use // significantly less memory. - ret = lzma_raw_encoder_init( + lzma_next_coder raw_encoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_raw_encoder_init( &raw_encoder, allocator, block->filters); if (ret == LZMA_OK) { @@ -227,15 +220,12 @@ block_encode_normal(lzma_block *block, lzma_allocator *allocator, } -extern LZMA_API(lzma_ret) -lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, +static lzma_ret +block_buffer_encode(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, - uint8_t *out, size_t *out_pos, size_t out_size) + uint8_t *out, size_t *out_pos, size_t out_size, + bool try_to_compress) { - size_t check_size; - lzma_ret ret; - size_t i; - // Validate the arguments. if (block == NULL || (in == NULL && in_size != 0) || out == NULL || out_pos == NULL || *out_pos > out_size) @@ -243,11 +233,11 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, // The contents of the structure may depend on the version so // check the version before validating the contents of *block. - if (block->version != 0) + if (block->version > 1) return LZMA_OPTIONS_ERROR; if ((unsigned int)(block->check) > LZMA_CHECK_ID_MAX - || block->filters == NULL) + || (try_to_compress && block->filters == NULL)) return LZMA_PROG_ERROR; if (!lzma_check_is_supported(block->check)) @@ -259,7 +249,7 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, out_size -= (out_size - *out_pos) & 3; // Get the size of the Check field. - check_size = lzma_check_size(block->check); + const size_t check_size = lzma_check_size(block->check); assert(check_size != UINT32_MAX); // Reserve space for the Check field. @@ -268,9 +258,19 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, out_size -= check_size; + // Initialize block->uncompressed_size and calculate the worst-case + // value for block->compressed_size. + block->uncompressed_size = in_size; + block->compressed_size = lzma2_bound(in_size); + if (block->compressed_size == 0) + return LZMA_DATA_ERROR; + // Do the actual compression. - ret = block_encode_normal(block, allocator, - in, in_size, out, out_pos, out_size); + lzma_ret ret = LZMA_BUF_ERROR; + if (try_to_compress) + ret = block_encode_normal(block, allocator, + in, in_size, out, out_pos, out_size); + if (ret != LZMA_OK) { // If the error was something else than output buffer // becoming full, return the error now. @@ -291,7 +291,7 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, // Block Padding. No buffer overflow here, because we already adjusted // out_size so that (out_size - out_start) is a multiple of four. // Thus, if the buffer is full, the loop body can never run. - for (i = (size_t)(block->compressed_size); i & 3; ++i) { + for (size_t i = (size_t)(block->compressed_size); i & 3; ++i) { assert(*out_pos < out_size); out[(*out_pos)++] = 0x00; } @@ -313,3 +313,25 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, return LZMA_OK; } + + +extern LZMA_API(lzma_ret) +lzma_block_buffer_encode(lzma_block *block, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + return block_buffer_encode(block, allocator, + in, in_size, out, out_pos, out_size, true); +} + + +extern LZMA_API(lzma_ret) +lzma_block_uncomp_encode(lzma_block *block, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + // It won't allocate any memory from heap so no need + // for lzma_allocator. + return block_buffer_encode(block, NULL, + in, in_size, out, out_pos, out_size, false); +} diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h new file mode 100644 index 0000000..653207f --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h @@ -0,0 +1,24 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_buffer_encoder.h +/// \brief Single-call .xz Block encoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BLOCK_BUFFER_ENCODER_H +#define LZMA_BLOCK_BUFFER_ENCODER_H + +#include "common.h" + + +/// uint64_t version of lzma_block_buffer_bound(). It is used by +/// stream_encoder_mt.c. Probably the original lzma_block_buffer_bound() +/// should have been 64-bit, but fixing it would break the ABI. +extern uint64_t lzma_block_buffer_bound64(uint64_t uncompressed_size); + +#endif diff --git a/Utilities/cmliblzma/liblzma/common/block_decoder.c b/Utilities/cmliblzma/liblzma/common/block_decoder.c index 35996e7..075bd27 100644 --- a/Utilities/cmliblzma/liblzma/common/block_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_decoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_CODE, SEQ_PADDING, @@ -45,7 +45,10 @@ struct lzma_coder_s { /// Check of the uncompressed data lzma_check_state check; -}; + + /// True if the integrity check won't be calculated and verified. + bool ignore_check; +} lzma_block_coder; static inline bool @@ -71,11 +74,13 @@ is_size_valid(lzma_vli size, lzma_vli reference) static lzma_ret -block_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +block_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_block_coder *coder = coder_ptr; + switch (coder->sequence) { case SEQ_CODE: { const size_t in_start = *in_pos; @@ -97,8 +102,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, coder->block->uncompressed_size)) return LZMA_DATA_ERROR; - lzma_check_update(&coder->check, coder->block->check, - out + out_start, out_used); + if (!coder->ignore_check) + lzma_check_update(&coder->check, coder->block->check, + out + out_start, out_used); if (ret != LZMA_STREAM_END) return ret; @@ -140,7 +146,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, if (coder->block->check == LZMA_CHECK_NONE) return LZMA_STREAM_END; - lzma_check_finish(&coder->check, coder->block->check); + if (!coder->ignore_check) + lzma_check_finish(&coder->check, coder->block->check); + coder->sequence = SEQ_CHECK; // Fall through @@ -155,7 +163,8 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, // Validate the Check only if we support it. // coder->check.buffer may be uninitialized // when the Check ID is not supported. - if (lzma_check_is_supported(coder->block->check) + if (!coder->ignore_check + && lzma_check_is_supported(coder->block->check) && memcmp(coder->block->raw_check, coder->check.buffer.u8, check_size) != 0) @@ -170,8 +179,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, static void -block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +block_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_block_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -179,7 +189,7 @@ block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_block_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_block *block) { lzma_next_coder_init(&lzma_block_decoder_init, next, allocator); @@ -191,27 +201,29 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, || !lzma_vli_is_valid(block->uncompressed_size)) return LZMA_PROG_ERROR; - // Allocate and initialize *next->coder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + // Allocate *next->coder if needed. + lzma_block_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_block_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &block_decode; next->end = &block_decoder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_CODE; - next->coder->block = block; - next->coder->compressed_size = 0; - next->coder->uncompressed_size = 0; + coder->sequence = SEQ_CODE; + coder->block = block; + coder->compressed_size = 0; + coder->uncompressed_size = 0; // If Compressed Size is not known, we calculate the maximum allowed // value so that encoded size of the Block (including Block Padding) // is still a valid VLI and a multiple of four. - next->coder->compressed_limit + coder->compressed_limit = block->compressed_size == LZMA_VLI_UNKNOWN ? (LZMA_VLI_MAX & ~LZMA_VLI_C(3)) - block->header_size @@ -221,11 +233,14 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, // Initialize the check. It's caller's problem if the Check ID is not // supported, and the Block decoder cannot verify the Check field. // Caller can test lzma_check_is_supported(block->check). - next->coder->check_pos = 0; - lzma_check_init(&next->coder->check, block->check); + coder->check_pos = 0; + lzma_check_init(&coder->check, block->check); + + coder->ignore_check = block->version >= 1 + ? block->ignore_check : false; // Initialize the filter chain. - return lzma_raw_decoder_init(&next->coder->next, allocator, + return lzma_raw_decoder_init(&coder->next, allocator, block->filters); } @@ -233,7 +248,7 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_block_decoder(lzma_stream *strm, lzma_block *block) { - lzma_next_strm_init1(lzma_block_decoder_init, strm, block); + lzma_next_strm_init(lzma_block_decoder_init, strm, block); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_decoder.h b/Utilities/cmliblzma/liblzma/common/block_decoder.h index 7da9df6..718c5ce 100644 --- a/Utilities/cmliblzma/liblzma/common/block_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/block_decoder.h @@ -17,6 +17,6 @@ extern lzma_ret lzma_block_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_block *block); + const lzma_allocator *allocator, lzma_block *block); #endif diff --git a/Utilities/cmliblzma/liblzma/common/block_encoder.c b/Utilities/cmliblzma/liblzma/common/block_encoder.c index ed74827..168846a 100644 --- a/Utilities/cmliblzma/liblzma/common/block_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_encoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { /// The filters in the chain; initialized with lzma_raw_decoder_init(). lzma_next_coder next; @@ -41,15 +41,17 @@ struct lzma_coder_s { /// Check of the uncompressed data lzma_check_state check; -}; +} lzma_block_coder; static lzma_ret -block_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +block_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_block_coder *coder = coder_ptr; + // Check that our amount of input stays in proper limits. if (LZMA_VLI_MAX - coder->uncompressed_size < in_size - *in_pos) return LZMA_DATA_ERROR; @@ -134,8 +136,9 @@ block_encode(lzma_coder *coder, lzma_allocator *allocator, static void -block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +block_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_block_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -143,10 +146,12 @@ block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -block_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +block_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_block_coder *coder = coder_ptr; + if (coder->sequence != SEQ_CODE) return LZMA_PROG_ERROR; @@ -156,7 +161,7 @@ block_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_block_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_block *block) { lzma_next_coder_init(&lzma_block_encoder_init, next, allocator); @@ -166,7 +171,7 @@ lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // The contents of the structure may depend on the version so // check the version first. - if (block->version != 0) + if (block->version > 1) return LZMA_OPTIONS_ERROR; // If the Check ID is not supported, we cannot calculate the check and @@ -178,37 +183,38 @@ lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_UNSUPPORTED_CHECK; // Allocate and initialize *next->coder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_block_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_block_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &block_encode; next->end = &block_encoder_end; next->update = &block_encoder_update; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_CODE; - next->coder->block = block; - next->coder->compressed_size = 0; - next->coder->uncompressed_size = 0; - next->coder->pos = 0; + coder->sequence = SEQ_CODE; + coder->block = block; + coder->compressed_size = 0; + coder->uncompressed_size = 0; + coder->pos = 0; // Initialize the check - lzma_check_init(&next->coder->check, block->check); + lzma_check_init(&coder->check, block->check); // Initialize the requested filters. - return lzma_raw_encoder_init(&next->coder->next, allocator, - block->filters); + return lzma_raw_encoder_init(&coder->next, allocator, block->filters); } extern LZMA_API(lzma_ret) lzma_block_encoder(lzma_stream *strm, lzma_block *block) { - lzma_next_strm_init1(lzma_block_encoder_init, strm, block); + lzma_next_strm_init(lzma_block_encoder_init, strm, block); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_encoder.h b/Utilities/cmliblzma/liblzma/common/block_encoder.h index b9eff0b..bd97c18 100644 --- a/Utilities/cmliblzma/liblzma/common/block_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/block_encoder.h @@ -42,6 +42,6 @@ extern lzma_ret lzma_block_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_block *block); + const lzma_allocator *allocator, lzma_block *block); #endif diff --git a/Utilities/cmliblzma/liblzma/common/block_header_decoder.c b/Utilities/cmliblzma/liblzma/common/block_header_decoder.c index f6e470e..1dd982f 100644 --- a/Utilities/cmliblzma/liblzma/common/block_header_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_header_decoder.c @@ -15,14 +15,12 @@ static void -free_properties(lzma_block *block, lzma_allocator *allocator) +free_properties(lzma_block *block, const lzma_allocator *allocator) { - size_t i; - // Free allocated filter options. The last array member is not // touched after the initialization in the beginning of // lzma_block_header_decode(), so we don't need to touch that here. - for (i = 0; i < LZMA_FILTERS_MAX; ++i) { + for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) { lzma_free(block->filters[i].options, allocator); block->filters[i].id = LZMA_VLI_UNKNOWN; block->filters[i].options = NULL; @@ -34,15 +32,8 @@ free_properties(lzma_block *block, lzma_allocator *allocator) extern LZMA_API(lzma_ret) lzma_block_header_decode(lzma_block *block, - lzma_allocator *allocator, const uint8_t *in) + const lzma_allocator *allocator, const uint8_t *in) { - const size_t filter_count = (in[1] & 3) + 1; - size_t in_size; - size_t i; - - // Start after the Block Header Size and Block Flags fields. - size_t in_pos = 2; - // NOTE: We consider the header to be corrupt not only when the // CRC32 doesn't match, but also when variable-length integers // are invalid or over 63 bits, or if the header is too small @@ -50,13 +41,21 @@ lzma_block_header_decode(lzma_block *block, // Initialize the filter options array. This way the caller can // safely free() the options even if an error occurs in this function. - for (i = 0; i <= LZMA_FILTERS_MAX; ++i) { + for (size_t i = 0; i <= LZMA_FILTERS_MAX; ++i) { block->filters[i].id = LZMA_VLI_UNKNOWN; block->filters[i].options = NULL; } - // Always zero for now. - block->version = 0; + // Versions 0 and 1 are supported. If a newer version was specified, + // we need to downgrade it. + if (block->version > 1) + block->version = 1; + + // This isn't a Block Header option, but since the decompressor will + // read it if version >= 1, it's better to initialize it here than + // to expect the caller to do it since in almost all cases this + // should be false. + block->ignore_check = false; // Validate Block Header Size and Check type. The caller must have // already set these, so it is a programming error if this test fails. @@ -65,7 +64,7 @@ lzma_block_header_decode(lzma_block *block, return LZMA_PROG_ERROR; // Exclude the CRC32 field. - in_size = block->header_size - 4; + const size_t in_size = block->header_size - 4; // Verify CRC32 if (lzma_crc32(in, in_size, 0) != unaligned_read32le(in + in_size)) @@ -75,6 +74,9 @@ lzma_block_header_decode(lzma_block *block, if (in[1] & 0x3C) return LZMA_OPTIONS_ERROR; + // Start after the Block Header Size and Block Flags fields. + size_t in_pos = 2; + // Compressed Size if (in[1] & 0x40) { return_if_error(lzma_vli_decode(&block->compressed_size, @@ -96,7 +98,8 @@ lzma_block_header_decode(lzma_block *block, block->uncompressed_size = LZMA_VLI_UNKNOWN; // Filter Flags - for (i = 0; i < filter_count; ++i) { + const size_t filter_count = (in[1] & 3) + 1; + for (size_t i = 0; i < filter_count; ++i) { const lzma_ret ret = lzma_filter_flags_decode( &block->filters[i], allocator, in, &in_pos, in_size); diff --git a/Utilities/cmliblzma/liblzma/common/block_header_encoder.c b/Utilities/cmliblzma/liblzma/common/block_header_encoder.c index 650295c..5c5f542 100644 --- a/Utilities/cmliblzma/liblzma/common/block_header_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_header_encoder.c @@ -17,14 +17,12 @@ extern LZMA_API(lzma_ret) lzma_block_header_size(lzma_block *block) { - size_t i; + if (block->version > 1) + return LZMA_OPTIONS_ERROR; // Block Header Size + Block Flags + CRC32. uint32_t size = 1 + 1 + 4; - if (block->version != 0) - return LZMA_OPTIONS_ERROR; - // Compressed Size if (block->compressed_size != LZMA_VLI_UNKNOWN) { const uint32_t add = lzma_vli_size(block->compressed_size); @@ -47,13 +45,12 @@ lzma_block_header_size(lzma_block *block) if (block->filters == NULL || block->filters[0].id == LZMA_VLI_UNKNOWN) return LZMA_PROG_ERROR; - for (i = 0; block->filters[i].id != LZMA_VLI_UNKNOWN; ++i) { - uint32_t add; - + for (size_t i = 0; block->filters[i].id != LZMA_VLI_UNKNOWN; ++i) { // Don't allow too many filters. if (i == LZMA_FILTERS_MAX) return LZMA_PROG_ERROR; + uint32_t add; return_if_error(lzma_filter_flags_size(&add, block->filters + i)); @@ -76,23 +73,20 @@ lzma_block_header_size(lzma_block *block) extern LZMA_API(lzma_ret) lzma_block_header_encode(const lzma_block *block, uint8_t *out) { - size_t out_size; - size_t out_pos = 2; - size_t filter_count = 0; - // Validate everything but filters. if (lzma_block_unpadded_size(block) == 0 || !lzma_vli_is_valid(block->uncompressed_size)) return LZMA_PROG_ERROR; // Indicate the size of the buffer _excluding_ the CRC32 field. - out_size = block->header_size - 4; + const size_t out_size = block->header_size - 4; // Store the Block Header Size. out[0] = out_size / 4; // We write Block Flags in pieces. out[1] = 0x00; + size_t out_pos = 2; // Compressed Size if (block->compressed_size != LZMA_VLI_UNKNOWN) { @@ -114,6 +108,7 @@ lzma_block_header_encode(const lzma_block *block, uint8_t *out) if (block->filters == NULL || block->filters[0].id == LZMA_VLI_UNKNOWN) return LZMA_PROG_ERROR; + size_t filter_count = 0; do { // There can be a maximum of four filters. if (filter_count == LZMA_FILTERS_MAX) diff --git a/Utilities/cmliblzma/liblzma/common/block_util.c b/Utilities/cmliblzma/liblzma/common/block_util.c index 4cd34d1..00c7fe8 100644 --- a/Utilities/cmliblzma/liblzma/common/block_util.c +++ b/Utilities/cmliblzma/liblzma/common/block_util.c @@ -17,14 +17,11 @@ extern LZMA_API(lzma_ret) lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) { - uint32_t container_size; - lzma_vli compressed_size; - // Validate everything but Uncompressed Size and filters. if (lzma_block_unpadded_size(block) == 0) return LZMA_PROG_ERROR; - container_size = block->header_size + const uint32_t container_size = block->header_size + lzma_check_size(block->check); // Validate that Compressed Size will be greater than zero. @@ -34,7 +31,7 @@ lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) // Calculate what Compressed Size is supposed to be. // If Compressed Size was present in Block Header, // compare that the new value matches it. - compressed_size = unpadded_size - container_size; + const lzma_vli compressed_size = unpadded_size - container_size; if (block->compressed_size != LZMA_VLI_UNKNOWN && block->compressed_size != compressed_size) return LZMA_DATA_ERROR; @@ -48,15 +45,13 @@ lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) extern LZMA_API(lzma_vli) lzma_block_unpadded_size(const lzma_block *block) { - lzma_vli unpadded_size; - // Validate the values that we are interested in i.e. all but // Uncompressed Size and the filters. // // NOTE: This function is used for validation too, so it is // essential that these checks are always done even if // Compressed Size is unknown. - if (block == NULL || block->version != 0 + if (block == NULL || block->version > 1 || block->header_size < LZMA_BLOCK_HEADER_SIZE_MIN || block->header_size > LZMA_BLOCK_HEADER_SIZE_MAX || (block->header_size & 3) @@ -71,7 +66,7 @@ lzma_block_unpadded_size(const lzma_block *block) return LZMA_VLI_UNKNOWN; // Calculate Unpadded Size and validate it. - unpadded_size = block->compressed_size + const lzma_vli unpadded_size = block->compressed_size + block->header_size + lzma_check_size(block->check); diff --git a/Utilities/cmliblzma/liblzma/common/common.c b/Utilities/cmliblzma/liblzma/common/common.c index 2e723c8..57e3f8e 100644 --- a/Utilities/cmliblzma/liblzma/common/common.c +++ b/Utilities/cmliblzma/liblzma/common/common.c @@ -36,14 +36,14 @@ lzma_version_string(void) /////////////////////// extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) -lzma_alloc(size_t size, lzma_allocator *allocator) +lzma_alloc(size_t size, const lzma_allocator *allocator) { - void *ptr; - // Some malloc() variants return NULL if called with size == 0. if (size == 0) size = 1; + void *ptr; + if (allocator != NULL && allocator->alloc != NULL) ptr = allocator->alloc(allocator->opaque, 1, size); else @@ -53,8 +53,29 @@ lzma_alloc(size_t size, lzma_allocator *allocator) } +extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) +lzma_alloc_zero(size_t size, const lzma_allocator *allocator) +{ + // Some calloc() variants return NULL if called with size == 0. + if (size == 0) + size = 1; + + void *ptr; + + if (allocator != NULL && allocator->alloc != NULL) { + ptr = allocator->alloc(allocator->opaque, 1, size); + if (ptr != NULL) + memzero(ptr, size); + } else { + ptr = calloc(1, size); + } + + return ptr; +} + + extern void -lzma_free(void *ptr, lzma_allocator *allocator) +lzma_free(void *ptr, const lzma_allocator *allocator) { if (allocator != NULL && allocator->free != NULL) allocator->free(allocator->opaque, ptr); @@ -70,9 +91,9 @@ lzma_free(void *ptr, lzma_allocator *allocator) ////////// extern size_t -lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size) +lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) { const size_t in_avail = in_size - *in_pos; const size_t out_avail = out_size - *out_pos; @@ -88,7 +109,7 @@ lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, extern lzma_ret -lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_next_filter_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { lzma_next_coder_init(filters[0].init, next, allocator); @@ -99,7 +120,7 @@ lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_next_filter_update(lzma_next_coder *next, lzma_allocator *allocator, +lzma_next_filter_update(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *reversed_filters) { // Check that the application isn't trying to change the Filter ID. @@ -117,7 +138,7 @@ lzma_next_filter_update(lzma_next_coder *next, lzma_allocator *allocator, extern void -lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator) +lzma_next_end(lzma_next_coder *next, const lzma_allocator *allocator) { if (next->init != (uintptr_t)(NULL)) { // To avoid tiny end functions that simply call @@ -156,10 +177,8 @@ lzma_strm_init(lzma_stream *strm) strm->internal->next = LZMA_NEXT_CODER_INIT; } - strm->internal->supported_actions[LZMA_RUN] = false; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = false; - strm->internal->supported_actions[LZMA_FULL_FLUSH] = false; - strm->internal->supported_actions[LZMA_FINISH] = false; + memzero(strm->internal->supported_actions, + sizeof(strm->internal->supported_actions)); strm->internal->sequence = ISEQ_RUN; strm->internal->allow_buf_error = false; @@ -173,16 +192,12 @@ lzma_strm_init(lzma_stream *strm) extern LZMA_API(lzma_ret) lzma_code(lzma_stream *strm, lzma_action action) { - size_t in_pos = 0; - size_t out_pos = 0; - lzma_ret ret; - // Sanity checks if ((strm->next_in == NULL && strm->avail_in != 0) || (strm->next_out == NULL && strm->avail_out != 0) || strm->internal == NULL || strm->internal->next.code == NULL - || (unsigned int)(action) > LZMA_FINISH + || (unsigned int)(action) > LZMA_ACTION_MAX || !strm->internal->supported_actions[action]) return LZMA_PROG_ERROR; @@ -217,6 +232,10 @@ lzma_code(lzma_stream *strm, lzma_action action) case LZMA_FINISH: strm->internal->sequence = ISEQ_FINISH; break; + + case LZMA_FULL_BARRIER: + strm->internal->sequence = ISEQ_FULL_BARRIER; + break; } break; @@ -244,6 +263,13 @@ lzma_code(lzma_stream *strm, lzma_action action) break; + case ISEQ_FULL_BARRIER: + if (action != LZMA_FULL_BARRIER + || strm->internal->avail_in != strm->avail_in) + return LZMA_PROG_ERROR; + + break; + case ISEQ_END: return LZMA_STREAM_END; @@ -252,7 +278,9 @@ lzma_code(lzma_stream *strm, lzma_action action) return LZMA_PROG_ERROR; } - ret = strm->internal->next.code( + size_t in_pos = 0; + size_t out_pos = 0; + lzma_ret ret = strm->internal->next.code( strm->internal->next.coder, strm->allocator, strm->next_in, &in_pos, strm->avail_in, strm->next_out, &out_pos, strm->avail_out, action); @@ -267,7 +295,9 @@ lzma_code(lzma_stream *strm, lzma_action action) strm->internal->avail_in = strm->avail_in; - switch (ret) { + // Cast is needed to silence a warning about LZMA_TIMED_OUT, which + // isn't part of lzma_ret enumeration. + switch ((unsigned int)(ret)) { case LZMA_OK: // Don't return LZMA_BUF_ERROR when it happens the first time. // This is to avoid returning LZMA_BUF_ERROR when avail_out @@ -283,9 +313,16 @@ lzma_code(lzma_stream *strm, lzma_action action) } break; + case LZMA_TIMED_OUT: + strm->internal->allow_buf_error = false; + ret = LZMA_OK; + break; + case LZMA_STREAM_END: if (strm->internal->sequence == ISEQ_SYNC_FLUSH - || strm->internal->sequence == ISEQ_FULL_FLUSH) + || strm->internal->sequence == ISEQ_FULL_FLUSH + || strm->internal->sequence + == ISEQ_FULL_BARRIER) strm->internal->sequence = ISEQ_RUN; else strm->internal->sequence = ISEQ_END; @@ -325,6 +362,22 @@ lzma_end(lzma_stream *strm) } +extern LZMA_API(void) +lzma_get_progress(lzma_stream *strm, + uint64_t *progress_in, uint64_t *progress_out) +{ + if (strm->internal->next.get_progress != NULL) { + strm->internal->next.get_progress(strm->internal->next.coder, + progress_in, progress_out); + } else { + *progress_in = strm->total_in; + *progress_out = strm->total_out; + } + + return; +} + + extern LZMA_API(lzma_check) lzma_get_check(const lzma_stream *strm) { @@ -382,8 +435,10 @@ lzma_memlimit_set(lzma_stream *strm, uint64_t new_memlimit) || strm->internal->next.memconfig == NULL) return LZMA_PROG_ERROR; - if (new_memlimit != 0 && new_memlimit < LZMA_MEMUSAGE_BASE) - return LZMA_MEMLIMIT_ERROR; + // Zero is a special value that cannot be used as an actual limit. + // If 0 was specified, use 1 instead. + if (new_memlimit == 0) + new_memlimit = 1; return strm->internal->next.memconfig(strm->internal->next.coder, &memusage, &old_memlimit, new_memlimit); diff --git a/Utilities/cmliblzma/liblzma/common/common.h b/Utilities/cmliblzma/liblzma/common/common.h index a6a2818..dde3ae0 100644 --- a/Utilities/cmliblzma/liblzma/common/common.h +++ b/Utilities/cmliblzma/liblzma/common/common.h @@ -48,6 +48,13 @@ #define LZMA_BUFFER_SIZE 4096 +/// Maximum number of worker threads within one multithreaded component. +/// The limit exists solely to make it simpler to prevent integer overflows +/// when allocating structures etc. This should be big enough for now... +/// the code won't scale anywhere close to this number anyway. +#define LZMA_THREADS_MAX 16384 + + /// Starting value for memory usage estimates. Instead of calculating size /// of _every_ structure and taking into account malloc() overhead etc., we /// add a base size to all memory usage estimates. It's not very accurate @@ -65,12 +72,20 @@ ( LZMA_TELL_NO_CHECK \ | LZMA_TELL_UNSUPPORTED_CHECK \ | LZMA_TELL_ANY_CHECK \ + | LZMA_IGNORE_CHECK \ | LZMA_CONCATENATED ) -/// Type of encoder/decoder specific data; the actual structure is defined -/// differently in different coders. -typedef struct lzma_coder_s lzma_coder; +/// Largest valid lzma_action value as unsigned integer. +#define LZMA_ACTION_MAX ((unsigned int)(LZMA_FULL_BARRIER)) + + +/// Special return value (lzma_ret) to indicate that a timeout was reached +/// and lzma_code() must not return LZMA_BUF_ERROR. This is converted to +/// LZMA_OK in lzma_code(). This is not in the lzma_ret enumeration because +/// there's no need to have it in the public API. +#define LZMA_TIMED_OUT 32 + typedef struct lzma_next_coder_s lzma_next_coder; @@ -79,7 +94,7 @@ typedef struct lzma_filter_info_s lzma_filter_info; /// Type of a function used to initialize a filter encoder or decoder typedef lzma_ret (*lzma_init_function)( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); /// Type of a function to do some kind of coding work (filters, Stream, @@ -87,15 +102,15 @@ typedef lzma_ret (*lzma_init_function)( /// input and output buffers, but for simplicity they still use this same /// function prototype. typedef lzma_ret (*lzma_code_function)( - lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, + void *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action); /// Type of a function to free the memory allocated for the coder typedef void (*lzma_end_function)( - lzma_coder *coder, lzma_allocator *allocator); + void *coder, const lzma_allocator *allocator); /// Raw coder validates and converts an array of lzma_filter structures to @@ -118,7 +133,7 @@ struct lzma_filter_info_s { /// Hold data and function pointers of the next filter in the chain. struct lzma_next_coder_s { /// Pointer to coder-specific data - lzma_coder *coder; + void *coder; /// Filter ID. This is LZMA_VLI_UNKNOWN when this structure doesn't /// point to a filter coder. @@ -138,35 +153,41 @@ struct lzma_next_coder_s { /// lzma_next_coder.coder. lzma_end_function end; + /// Pointer to a function to get progress information. If this is NULL, + /// lzma_stream.total_in and .total_out are used instead. + void (*get_progress)(void *coder, + uint64_t *progress_in, uint64_t *progress_out); + /// Pointer to function to return the type of the integrity check. /// Most coders won't support this. - lzma_check (*get_check)(const lzma_coder *coder); + lzma_check (*get_check)(const void *coder); /// Pointer to function to get and/or change the memory usage limit. /// If new_memlimit == 0, the limit is not changed. - lzma_ret (*memconfig)(lzma_coder *coder, uint64_t *memusage, + lzma_ret (*memconfig)(void *coder, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit); /// Update the filter-specific options or the whole filter chain /// in the encoder. - lzma_ret (*update)(lzma_coder *coder, lzma_allocator *allocator, + lzma_ret (*update)(void *coder, const lzma_allocator *allocator, const lzma_filter *filters, const lzma_filter *reversed_filters); }; -/// Constant to initialize lzma_next_coder structure -static const lzma_next_coder LZMA_NEXT_CODER_INIT = - { - NULL, - LZMA_VLI_UNKNOWN, - (uintptr_t)(NULL), - NULL, - NULL, - NULL, - NULL, - NULL, - }; +/// Macro to initialize lzma_next_coder structure +#define LZMA_NEXT_CODER_INIT \ + (lzma_next_coder){ \ + .coder = NULL, \ + .init = (uintptr_t)(NULL), \ + .id = LZMA_VLI_UNKNOWN, \ + .code = NULL, \ + .end = NULL, \ + .get_progress = NULL, \ + .get_check = NULL, \ + .memconfig = NULL, \ + .update = NULL, \ + } /// Internal data for lzma_strm_init, lzma_code, and lzma_end. A pointer to @@ -184,6 +205,7 @@ struct lzma_internal_s { ISEQ_SYNC_FLUSH, ISEQ_FULL_FLUSH, ISEQ_FINISH, + ISEQ_FULL_BARRIER, ISEQ_END, ISEQ_ERROR, } sequence; @@ -194,7 +216,7 @@ struct lzma_internal_s { size_t avail_in; /// Indicates which lzma_action values are allowed by next.code. - bool supported_actions[4]; + bool supported_actions[LZMA_ACTION_MAX + 1]; /// If true, lzma_code will return LZMA_BUF_ERROR if no progress was /// made (no input consumed and no output produced by next.code). @@ -203,15 +225,21 @@ struct lzma_internal_s { /// Allocates memory -extern void *lzma_alloc(size_t size, lzma_allocator *allocator) +extern void *lzma_alloc(size_t size, const lzma_allocator *allocator) lzma_attribute((__malloc__)) lzma_attr_alloc_size(1); +/// Allocates memory and zeroes it (like calloc()). This can be faster +/// than lzma_alloc() + memzero() while being backward compatible with +/// custom allocators. +extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) + lzma_alloc_zero(size_t size, const lzma_allocator *allocator); + /// Frees memory -extern void lzma_free(void *ptr, lzma_allocator *allocator); +extern void lzma_free(void *ptr, const lzma_allocator *allocator); /// Allocates strm->internal if it is NULL, and initializes *strm and -/// strm->internal. This function is only called via lzma_next_strm_init2 macro. +/// strm->internal. This function is only called via lzma_next_strm_init macro. extern lzma_ret lzma_strm_init(lzma_stream *strm); /// Initializes the next filter in the chain, if any. This takes care of @@ -219,24 +247,26 @@ extern lzma_ret lzma_strm_init(lzma_stream *strm); /// than the filter being initialized now. This way the actual filter /// initialization functions don't need to use lzma_next_coder_init macro. extern lzma_ret lzma_next_filter_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); /// Update the next filter in the chain, if any. This checks that /// the application is not trying to change the Filter IDs. extern lzma_ret lzma_next_filter_update( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *reversed_filters); /// Frees the memory allocated for next->coder either using next->end or, /// if next->end is NULL, using lzma_free. -extern void lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator); +extern void lzma_next_end(lzma_next_coder *next, + const lzma_allocator *allocator); /// Copy as much data as possible from in[] to out[] and update *in_pos /// and *out_pos accordingly. Returns the number of bytes copied. -extern size_t lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size); +extern size_t lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); /// \brief Return if expression doesn't evaluate to LZMA_OK @@ -269,37 +299,15 @@ do { \ /// (The function being called will use lzma_next_coder_init()). If /// initialization fails, memory that wasn't freed by func() is freed /// along strm->internal. -#define lzma_next_strm_init1(func, strm, arg1) \ +#define lzma_next_strm_init(func, strm, ...) \ do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ -} while (0) - -#define lzma_next_strm_init2(func, strm, arg1, arg2) \ -do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1, arg2); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ -} while (0) - -#define lzma_next_strm_init3(func, strm, arg1, arg2, arg3) \ -do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1, arg2, arg3); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ + return_if_error(lzma_strm_init(strm)); \ + const lzma_ret ret_ = func(&(strm)->internal->next, \ + (strm)->allocator, __VA_ARGS__); \ + if (ret_ != LZMA_OK) { \ + lzma_end(strm); \ + return ret_; \ + } \ } while (0) #endif diff --git a/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c index c4be34c..48eb56f 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c @@ -15,8 +15,8 @@ extern LZMA_API(lzma_ret) lzma_easy_buffer_encode(uint32_t preset, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, - uint8_t *out, size_t *out_pos, size_t out_size) + const lzma_allocator *allocator, const uint8_t *in, + size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { lzma_options_easy opt_easy; if (lzma_easy_preset(&opt_easy, preset)) diff --git a/Utilities/cmliblzma/liblzma/common/easy_encoder.c b/Utilities/cmliblzma/liblzma/common/easy_encoder.c index d13ccd7..5cb492d 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/easy_encoder.c @@ -11,7 +11,6 @@ /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" -#include "stream_encoder.h" extern LZMA_API(lzma_ret) diff --git a/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c index 65665c1..6620986 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c @@ -14,30 +14,27 @@ extern LZMA_API(lzma_ret) -lzma_raw_buffer_decode(const lzma_filter *filters, lzma_allocator *allocator, +lzma_raw_buffer_decode( + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder next = LZMA_NEXT_CODER_INIT; - size_t in_start; - size_t out_start; - lzma_ret ret; - // Validate what isn't validated later in filter_common.c. if (in == NULL || in_pos == NULL || *in_pos > in_size || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; // Initialize the decoer. + lzma_next_coder next = LZMA_NEXT_CODER_INIT; return_if_error(lzma_raw_decoder_init(&next, allocator, filters)); // Store the positions so that we can restore them if something // goes wrong. - in_start = *in_pos; - out_start = *out_pos; + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; // Do the actual decoding and free decoder's memory. - ret = next.code(next.coder, allocator, in, in_pos, in_size, + lzma_ret ret = next.code(next.coder, allocator, in, in_pos, in_size, out, out_pos, out_size, LZMA_FINISH); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c index b23329f..dda18e3 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c @@ -14,29 +14,27 @@ extern LZMA_API(lzma_ret) -lzma_raw_buffer_encode(const lzma_filter *filters, lzma_allocator *allocator, - const uint8_t *in, size_t in_size, uint8_t *out, - size_t *out_pos, size_t out_size) +lzma_raw_buffer_encode( + const lzma_filter *filters, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder next = LZMA_NEXT_CODER_INIT; - size_t out_start; - size_t in_pos = 0; - lzma_ret ret; - // Validate what isn't validated later in filter_common.c. if ((in == NULL && in_size != 0) || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; // Initialize the encoder + lzma_next_coder next = LZMA_NEXT_CODER_INIT; return_if_error(lzma_raw_encoder_init(&next, allocator, filters)); // Store the output position so that we can restore it if // something goes wrong. - out_start = *out_pos; + const size_t out_start = *out_pos; // Do the actual encoding and free coder's memory. - ret = next.code(next.coder, allocator, in, &in_pos, in_size, + size_t in_pos = 0; + lzma_ret ret = next.code(next.coder, allocator, in, &in_pos, in_size, out, out_pos, out_size, LZMA_FINISH); lzma_next_end(&next, allocator); diff --git a/Utilities/cmliblzma/liblzma/common/filter_common.c b/Utilities/cmliblzma/liblzma/common/filter_common.c index d2b9e08..9ad5d5d 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_common.c +++ b/Utilities/cmliblzma/liblzma/common/filter_common.c @@ -36,101 +36,100 @@ static const struct { } features[] = { #if defined (HAVE_ENCODER_LZMA1) || defined(HAVE_DECODER_LZMA1) { - LZMA_FILTER_LZMA1, - sizeof(lzma_options_lzma), - false, - true, - true, + .id = LZMA_FILTER_LZMA1, + .options_size = sizeof(lzma_options_lzma), + .non_last_ok = false, + .last_ok = true, + .changes_size = true, }, #endif #if defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2) { - LZMA_FILTER_LZMA2, - sizeof(lzma_options_lzma), - false, - true, - true, + .id = LZMA_FILTER_LZMA2, + .options_size = sizeof(lzma_options_lzma), + .non_last_ok = false, + .last_ok = true, + .changes_size = true, }, #endif #if defined(HAVE_ENCODER_X86) || defined(HAVE_DECODER_X86) { - LZMA_FILTER_X86, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_X86, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_POWERPC) || defined(HAVE_DECODER_POWERPC) { - LZMA_FILTER_POWERPC, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_POWERPC, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_IA64) || defined(HAVE_DECODER_IA64) { - LZMA_FILTER_IA64, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_IA64, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_ARM) || defined(HAVE_DECODER_ARM) { - LZMA_FILTER_ARM, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_ARM, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_ARMTHUMB) || defined(HAVE_DECODER_ARMTHUMB) { - LZMA_FILTER_ARMTHUMB, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_ARMTHUMB, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_SPARC) || defined(HAVE_DECODER_SPARC) { - LZMA_FILTER_SPARC, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_SPARC, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) { - LZMA_FILTER_DELTA, - sizeof(lzma_options_delta), - true, - false, - false, + .id = LZMA_FILTER_DELTA, + .options_size = sizeof(lzma_options_delta), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif { - LZMA_VLI_UNKNOWN + .id = LZMA_VLI_UNKNOWN } }; extern LZMA_API(lzma_ret) lzma_filters_copy(const lzma_filter *src, lzma_filter *dest, - lzma_allocator *allocator) + const lzma_allocator *allocator) { - size_t i; - lzma_ret ret; - if (src == NULL || dest == NULL) return LZMA_PROG_ERROR; + lzma_ret ret; + size_t i; for (i = 0; src[i].id != LZMA_VLI_UNKNOWN; ++i) { // There must be a maximum of four filters plus // the array terminator. @@ -194,6 +193,10 @@ error: static lzma_ret validate_chain(const lzma_filter *filters, size_t *count) { + // There must be at least one filter. + if (filters == NULL || filters[0].id == LZMA_VLI_UNKNOWN) + return LZMA_PROG_ERROR; + // Number of non-last filters that may change the size of the data // significantly (that is, more than 1-2 % or so). size_t changes_size_count = 0; @@ -207,11 +210,6 @@ validate_chain(const lzma_filter *filters, size_t *count) bool last_ok = false; size_t i = 0; - - // There must be at least one filter. - if (filters == NULL || filters[0].id == LZMA_VLI_UNKNOWN) - return LZMA_PROG_ERROR; - do { size_t j; for (j = 0; filters[i].id != features[j].id; ++j) @@ -241,21 +239,18 @@ validate_chain(const lzma_filter *filters, size_t *count) extern lzma_ret -lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options, lzma_filter_find coder_find, bool is_encoder) { - lzma_filter_info filters[LZMA_FILTERS_MAX + 1]; - size_t count; - size_t i; - lzma_ret ret; - // Do some basic validation and get the number of filters. + size_t count; return_if_error(validate_chain(options, &count)); // Set the filter functions and copy the options pointer. + lzma_filter_info filters[LZMA_FILTERS_MAX + 1]; if (is_encoder) { - for (i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { // The order of the filters is reversed in the // encoder. It allows more efficient handling // of the uncompressed data. @@ -271,7 +266,7 @@ lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, filters[j].options = options[i].options; } } else { - for (i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { const lzma_filter_coder *const fc = coder_find(options[i].id); if (fc == NULL || fc->init == NULL) @@ -288,7 +283,7 @@ lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, filters[count].init = NULL; // Initialize the filters. - ret = lzma_next_filter_init(next, allocator, filters); + const lzma_ret ret = lzma_next_filter_init(next, allocator, filters); if (ret != LZMA_OK) lzma_next_end(next, allocator); @@ -300,9 +295,6 @@ extern uint64_t lzma_raw_coder_memusage(lzma_filter_find coder_find, const lzma_filter *filters) { - uint64_t total = 0; - size_t i = 0; - // The chain has to have at least one filter. { size_t tmp; @@ -310,6 +302,9 @@ lzma_raw_coder_memusage(lzma_filter_find coder_find, return UINT64_MAX; } + uint64_t total = 0; + size_t i = 0; + do { const lzma_filter_coder *const fc = coder_find(filters[i].id); diff --git a/Utilities/cmliblzma/liblzma/common/filter_common.h b/Utilities/cmliblzma/liblzma/common/filter_common.h index cd61fc0..42a26a2 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_common.h +++ b/Utilities/cmliblzma/liblzma/common/filter_common.h @@ -36,7 +36,7 @@ typedef const lzma_filter_coder *(*lzma_filter_find)(lzma_vli id); extern lzma_ret lzma_raw_coder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters, lzma_filter_find coder_find, bool is_encoder); diff --git a/Utilities/cmliblzma/liblzma/common/filter_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_decoder.c index cce2b30..c75b0a8 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_decoder.c @@ -35,7 +35,8 @@ typedef struct { /// \return - LZMA_OK: Properties decoded successfully. /// - LZMA_OPTIONS_ERROR: Unsupported properties /// - LZMA_MEM_ERROR: Memory allocation failed. - lzma_ret (*props_decode)(void **options, lzma_allocator *allocator, + lzma_ret (*props_decode)( + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); } lzma_filter_decoder; @@ -44,74 +45,74 @@ typedef struct { static const lzma_filter_decoder decoders[] = { #ifdef HAVE_DECODER_LZMA1 { - LZMA_FILTER_LZMA1, - &lzma_lzma_decoder_init, - &lzma_lzma_decoder_memusage, - &lzma_lzma_props_decode, + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_decoder_init, + .memusage = &lzma_lzma_decoder_memusage, + .props_decode = &lzma_lzma_props_decode, }, #endif #ifdef HAVE_DECODER_LZMA2 { - LZMA_FILTER_LZMA2, - &lzma_lzma2_decoder_init, - &lzma_lzma2_decoder_memusage, - &lzma_lzma2_props_decode, + .id = LZMA_FILTER_LZMA2, + .init = &lzma_lzma2_decoder_init, + .memusage = &lzma_lzma2_decoder_memusage, + .props_decode = &lzma_lzma2_props_decode, }, #endif #ifdef HAVE_DECODER_X86 { - LZMA_FILTER_X86, - &lzma_simple_x86_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_X86, + .init = &lzma_simple_x86_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_POWERPC { - LZMA_FILTER_POWERPC, - &lzma_simple_powerpc_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_POWERPC, + .init = &lzma_simple_powerpc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_IA64 { - LZMA_FILTER_IA64, - &lzma_simple_ia64_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_IA64, + .init = &lzma_simple_ia64_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_ARM { - LZMA_FILTER_ARM, - &lzma_simple_arm_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_ARM, + .init = &lzma_simple_arm_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_ARMTHUMB { - LZMA_FILTER_ARMTHUMB, - &lzma_simple_armthumb_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_ARMTHUMB, + .init = &lzma_simple_armthumb_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_SPARC { - LZMA_FILTER_SPARC, - &lzma_simple_sparc_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_SPARC, + .init = &lzma_simple_sparc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_DELTA { - LZMA_FILTER_DELTA, - &lzma_delta_decoder_init, - &lzma_delta_coder_memusage, - &lzma_delta_props_decode, + .id = LZMA_FILTER_DELTA, + .init = &lzma_delta_decoder_init, + .memusage = &lzma_delta_coder_memusage, + .props_decode = &lzma_delta_props_decode, }, #endif }; @@ -120,8 +121,7 @@ static const lzma_filter_decoder decoders[] = { static const lzma_filter_decoder * decoder_find(lzma_vli id) { - size_t i; - for (i = 0; i < ARRAY_SIZE(decoders); ++i) + for (size_t i = 0; i < ARRAY_SIZE(decoders); ++i) if (decoders[i].id == id) return decoders + i; @@ -137,7 +137,7 @@ lzma_filter_decoder_is_supported(lzma_vli id) extern lzma_ret -lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options) { return lzma_raw_coder_init(next, allocator, @@ -148,7 +148,7 @@ lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_raw_decoder(lzma_stream *strm, const lzma_filter *options) { - lzma_next_strm_init1(lzma_raw_decoder_init, strm, options); + lzma_next_strm_init(lzma_raw_decoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -166,14 +166,13 @@ lzma_raw_decoder_memusage(const lzma_filter *filters) extern LZMA_API(lzma_ret) -lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator, +lzma_properties_decode(lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - const lzma_filter_decoder *const fd = decoder_find(filter->id); - // Make it always NULL so that the caller can always safely free() it. filter->options = NULL; + const lzma_filter_decoder *const fd = decoder_find(filter->id); if (fd == NULL) return LZMA_OPTIONS_ERROR; diff --git a/Utilities/cmliblzma/liblzma/common/filter_decoder.h b/Utilities/cmliblzma/liblzma/common/filter_decoder.h index d5c68bd..a2e255f 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/filter_decoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_raw_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options); #endif diff --git a/Utilities/cmliblzma/liblzma/common/filter_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_encoder.c index 9fdb100..c5d8f39 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_encoder.c @@ -30,11 +30,11 @@ typedef struct { /// invalid, UINT64_MAX is returned. uint64_t (*memusage)(const void *options); - /// Calculates the minimum sane size for Blocks (or other types of - /// chunks) to which the input data can be split to make - /// multithreaded encoding possible. If this is NULL, it is assumed - /// that the encoder is fast enough with single thread. - lzma_vli (*chunk_size)(const void *options); + /// Calculates the recommended Uncompressed Size for .xz Blocks to + /// which the input data can be split to make multithreaded + /// encoding possible. If this is NULL, it is assumed that + /// the encoder is fast enough with single thread. + uint64_t (*block_size)(const void *options); /// Tells the size of the Filter Properties field. If options are /// invalid, UINT32_MAX is returned. If this is NULL, props_size_fixed @@ -56,101 +56,95 @@ typedef struct { static const lzma_filter_encoder encoders[] = { #ifdef HAVE_ENCODER_LZMA1 { - LZMA_FILTER_LZMA1, - &lzma_lzma_encoder_init, - &lzma_lzma_encoder_memusage, - NULL, // FIXME - NULL, - 5, - &lzma_lzma_props_encode, + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_encoder_init, + .memusage = &lzma_lzma_encoder_memusage, + .block_size = NULL, // FIXME + .props_size_get = NULL, + .props_size_fixed = 5, + .props_encode = &lzma_lzma_props_encode, }, #endif #ifdef HAVE_ENCODER_LZMA2 { - LZMA_FILTER_LZMA2, - &lzma_lzma2_encoder_init, - &lzma_lzma2_encoder_memusage, - NULL, // FIXME - NULL, - 1, - &lzma_lzma2_props_encode, + .id = LZMA_FILTER_LZMA2, + .init = &lzma_lzma2_encoder_init, + .memusage = &lzma_lzma2_encoder_memusage, + .block_size = &lzma_lzma2_block_size, // FIXME + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_lzma2_props_encode, }, #endif #ifdef HAVE_ENCODER_X86 { - LZMA_FILTER_X86, - &lzma_simple_x86_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_X86, + .init = &lzma_simple_x86_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_POWERPC { - LZMA_FILTER_POWERPC, - &lzma_simple_powerpc_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_POWERPC, + .init = &lzma_simple_powerpc_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_IA64 { - LZMA_FILTER_IA64, - &lzma_simple_ia64_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_IA64, + .init = &lzma_simple_ia64_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_ARM { - LZMA_FILTER_ARM, - &lzma_simple_arm_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_ARM, + .init = &lzma_simple_arm_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_ARMTHUMB { - LZMA_FILTER_ARMTHUMB, - &lzma_simple_armthumb_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_ARMTHUMB, + .init = &lzma_simple_armthumb_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_SPARC { - LZMA_FILTER_SPARC, - &lzma_simple_sparc_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_SPARC, + .init = &lzma_simple_sparc_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_DELTA { - LZMA_FILTER_DELTA, - &lzma_delta_encoder_init, - &lzma_delta_coder_memusage, - NULL, - NULL, - 1, - &lzma_delta_props_encode, + .id = LZMA_FILTER_DELTA, + .init = &lzma_delta_encoder_init, + .memusage = &lzma_delta_coder_memusage, + .block_size = NULL, + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_delta_props_encode, }, #endif }; @@ -159,8 +153,7 @@ static const lzma_filter_encoder encoders[] = { static const lzma_filter_encoder * encoder_find(lzma_vli id) { - size_t i; - for (i = 0; i < ARRAY_SIZE(encoders); ++i) + for (size_t i = 0; i < ARRAY_SIZE(encoders); ++i) if (encoders[i].id == id) return encoders + i; @@ -178,10 +171,6 @@ lzma_filter_encoder_is_supported(lzma_vli id) extern LZMA_API(lzma_ret) lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) { - size_t i; - size_t count = 1; - lzma_filter reversed_filters[LZMA_FILTERS_MAX + 1]; - if (strm->internal->next.update == NULL) return LZMA_PROG_ERROR; @@ -191,10 +180,12 @@ lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) // The actual filter chain in the encoder is reversed. Some things // still want the normal order chain, so we provide both. + size_t count = 1; while (filters[count].id != LZMA_VLI_UNKNOWN) ++count; - for (i = 0; i < count; ++i) + lzma_filter reversed_filters[LZMA_FILTERS_MAX + 1]; + for (size_t i = 0; i < count; ++i) reversed_filters[count - i - 1] = filters[i]; reversed_filters[count].id = LZMA_VLI_UNKNOWN; @@ -205,7 +196,7 @@ lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) extern lzma_ret -lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options) { return lzma_raw_coder_init(next, allocator, @@ -216,7 +207,7 @@ lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options) { - lzma_next_strm_init3(lzma_raw_coder_init, strm, options, + lzma_next_strm_init(lzma_raw_coder_init, strm, options, (lzma_filter_find)(&encoder_find), true); strm->internal->supported_actions[LZMA_RUN] = true; @@ -235,20 +226,19 @@ lzma_raw_encoder_memusage(const lzma_filter *filters) } -/* -extern LZMA_API(lzma_vli) -lzma_chunk_size(const lzma_filter *filters) +extern uint64_t +lzma_mt_block_size(const lzma_filter *filters) { - lzma_vli max = 0; + uint64_t max = 0; for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { const lzma_filter_encoder *const fe = encoder_find(filters[i].id); - if (fe->chunk_size != NULL) { - const lzma_vli size - = fe->chunk_size(filters[i].options); - if (size == LZMA_VLI_UNKNOWN) - return LZMA_VLI_UNKNOWN; + if (fe->block_size != NULL) { + const uint64_t size + = fe->block_size(filters[i].options); + if (size == 0) + return 0; if (size > max) max = size; @@ -257,7 +247,6 @@ lzma_chunk_size(const lzma_filter *filters) return max; } -*/ extern LZMA_API(lzma_ret) diff --git a/Utilities/cmliblzma/liblzma/common/filter_encoder.h b/Utilities/cmliblzma/liblzma/common/filter_encoder.h index 5bc137f..f1d5683 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/filter_encoder.h @@ -16,12 +16,12 @@ #include "common.h" -// FIXME: Might become a part of the public API once finished. -// extern lzma_vli lzma_chunk_size(const lzma_filter *filters); +// FIXME: Might become a part of the public API. +extern uint64_t lzma_mt_block_size(const lzma_filter *filters); extern lzma_ret lzma_raw_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c index aa2dbd5..ddfb085 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c @@ -15,12 +15,9 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { - lzma_vli props_size; - lzma_ret ret; - // Set the pointer to NULL so the caller can always safely free it. filter->options = NULL; @@ -32,6 +29,7 @@ lzma_filter_flags_decode( return LZMA_DATA_ERROR; // Size of Properties + lzma_vli props_size; return_if_error(lzma_vli_decode(&props_size, NULL, in, in_pos, in_size)); @@ -39,7 +37,7 @@ lzma_filter_flags_decode( if (in_size - *in_pos < props_size) return LZMA_DATA_ERROR; - ret = lzma_properties_decode( + const lzma_ret ret = lzma_properties_decode( filter, allocator, in + *in_pos, props_size); *in_pos += props_size; diff --git a/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c index 755c407..d110566 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c @@ -31,8 +31,6 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filter, uint8_t *out, size_t *out_pos, size_t out_size) { - uint32_t props_size; - // Filter ID if (filter->id >= LZMA_FILTER_RESERVED_START) return LZMA_PROG_ERROR; @@ -41,6 +39,7 @@ lzma_filter_flags_encode(const lzma_filter *filter, out, out_pos, out_size)); // Size of Properties + uint32_t props_size; return_if_error(lzma_properties_size(&props_size, filter)); return_if_error(lzma_vli_encode(props_size, NULL, out, out_pos, out_size)); diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder.h b/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c index 46a7aed..f468366 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file stream_encoder.h -/// \brief Encodes .xz Streams +/// \file hardware_cputhreads.c +/// \brief Get the number of CPU threads or cores // // Author: Lasse Collin // @@ -10,14 +10,13 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_STREAM_ENCODER_H -#define LZMA_STREAM_ENCODER_H - #include "common.h" +#include "tuklib_cpucores.h" -extern lzma_ret lzma_stream_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, - const lzma_filter *filters, lzma_check check); -#endif +extern LZMA_API(uint32_t) +lzma_cputhreads(void) +{ + return tuklib_cpucores(); +} diff --git a/Utilities/cmliblzma/liblzma/common/index.c b/Utilities/cmliblzma/liblzma/common/index.c index 26135d2..26e4e51 100644 --- a/Utilities/cmliblzma/liblzma/common/index.c +++ b/Utilities/cmliblzma/liblzma/common/index.c @@ -191,8 +191,8 @@ index_tree_init(index_tree *tree) /// Helper for index_tree_end() static void -index_tree_node_end(index_tree_node *node, lzma_allocator *allocator, - void (*free_func)(void *node, lzma_allocator *allocator)) +index_tree_node_end(index_tree_node *node, const lzma_allocator *allocator, + void (*free_func)(void *node, const lzma_allocator *allocator)) { // The tree won't ever be very huge, so recursion should be fine. // 20 levels in the tree is likely quite a lot already in practice. @@ -202,22 +202,21 @@ index_tree_node_end(index_tree_node *node, lzma_allocator *allocator, if (node->right != NULL) index_tree_node_end(node->right, allocator, free_func); - if (free_func != NULL) - free_func(node, allocator); - - lzma_free(node, allocator); + free_func(node, allocator); return; } -/// Free the meory allocated for a tree. If free_func is not NULL, -/// it is called on each node before freeing the node. This is used -/// to free the Record groups from each index_stream before freeing -/// the index_stream itself. +/// Free the memory allocated for a tree. Each node is freed using the +/// given free_func which is either &lzma_free or &index_stream_end. +/// The latter is used to free the Record groups from each index_stream +/// before freeing the index_stream itself. static void -index_tree_end(index_tree *tree, lzma_allocator *allocator, - void (*free_func)(void *node, lzma_allocator *allocator)) +index_tree_end(index_tree *tree, const lzma_allocator *allocator, + void (*free_func)(void *node, const lzma_allocator *allocator)) { + assert(free_func != NULL); + if (tree->root != NULL) index_tree_node_end(tree->root, allocator, free_func); @@ -230,7 +229,6 @@ index_tree_end(index_tree *tree, lzma_allocator *allocator, static void index_tree_append(index_tree *tree, index_tree_node *node) { - uint32_t up; node->parent = tree->rightmost; node->left = NULL; node->right = NULL; @@ -259,10 +257,8 @@ index_tree_append(index_tree *tree, index_tree_node *node) // and thus know the state of the tree just by looking at the node // count. From the node count we can calculate how many steps to go // up in the tree to find the rotation root. - up = tree->count ^ (UINT32_C(1) << bsr32(tree->count)); + uint32_t up = tree->count ^ (UINT32_C(1) << bsr32(tree->count)); if (up != 0) { - index_tree_node *pivot; - // Locate the root node for the rotation. up = ctz32(tree->count) + 2; do { @@ -270,7 +266,7 @@ index_tree_append(index_tree *tree, index_tree_node *node) } while (--up > 0); // Rotate left using node as the rotation root. - pivot = node->right; + index_tree_node *pivot = node->right; if (node->parent == NULL) { tree->root = pivot; @@ -342,8 +338,8 @@ index_tree_locate(const index_tree *tree, lzma_vli target) /// Allocate and initialize a new Stream using the given base offsets. static index_stream * index_stream_init(lzma_vli compressed_base, lzma_vli uncompressed_base, - lzma_vli stream_number, lzma_vli block_number_base, - lzma_allocator *allocator) + uint32_t stream_number, lzma_vli block_number_base, + const lzma_allocator *allocator) { index_stream *s = lzma_alloc(sizeof(index_stream), allocator); if (s == NULL) @@ -371,16 +367,17 @@ index_stream_init(lzma_vli compressed_base, lzma_vli uncompressed_base, /// Free the memory allocated for a Stream and its Record groups. static void -index_stream_end(void *node, lzma_allocator *allocator) +index_stream_end(void *node, const lzma_allocator *allocator) { index_stream *s = node; - index_tree_end(&s->groups, allocator, NULL); + index_tree_end(&s->groups, allocator, &lzma_free); + lzma_free(s, allocator); return; } static lzma_index * -index_init_plain(lzma_allocator *allocator) +index_init_plain(const lzma_allocator *allocator) { lzma_index *i = lzma_alloc(sizeof(lzma_index), allocator); if (i != NULL) { @@ -398,15 +395,13 @@ index_init_plain(lzma_allocator *allocator) extern LZMA_API(lzma_index *) -lzma_index_init(lzma_allocator *allocator) +lzma_index_init(const lzma_allocator *allocator) { - index_stream *s; - lzma_index *i = index_init_plain(allocator); if (i == NULL) return NULL; - s = index_stream_init(0, 0, 1, 0, allocator); + index_stream *s = index_stream_init(0, 0, 1, 0, allocator); if (s == NULL) { lzma_free(i, allocator); return NULL; @@ -419,7 +414,7 @@ lzma_index_init(lzma_allocator *allocator) extern LZMA_API(void) -lzma_index_end(lzma_index *i, lzma_allocator *allocator) +lzma_index_end(lzma_index *i, const lzma_allocator *allocator) { // NOTE: If you modify this function, check also the bottom // of lzma_index_cat(). @@ -605,8 +600,6 @@ lzma_index_padding_size(const lzma_index *i) extern LZMA_API(lzma_ret) lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) { - index_stream *s; - if (i == NULL || stream_flags == NULL) return LZMA_PROG_ERROR; @@ -614,7 +607,7 @@ lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) return_if_error(lzma_stream_flags_compare( stream_flags, stream_flags)); - s = (index_stream *)(i->streams.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); s->stream_flags = *stream_flags; return LZMA_OK; @@ -624,17 +617,14 @@ lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) extern LZMA_API(lzma_ret) lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding) { - index_stream *s; - lzma_vli old_stream_padding; - if (i == NULL || stream_padding > LZMA_VLI_MAX || (stream_padding & 3) != 0) return LZMA_PROG_ERROR; - s = (index_stream *)(i->streams.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); // Check that the new value won't make the file grow too big. - old_stream_padding = s->stream_padding; + const lzma_vli old_stream_padding = s->stream_padding; s->stream_padding = 0; if (lzma_index_file_size(i) + stream_padding > LZMA_VLI_MAX) { s->stream_padding = old_stream_padding; @@ -647,29 +637,23 @@ lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding) extern LZMA_API(lzma_ret) -lzma_index_append(lzma_index *i, lzma_allocator *allocator, +lzma_index_append(lzma_index *i, const lzma_allocator *allocator, lzma_vli unpadded_size, lzma_vli uncompressed_size) { - index_stream *s; - index_group *g; - lzma_vli compressed_base; - lzma_vli uncompressed_base; - uint32_t index_list_size_add; - // Validate. if (i == NULL || unpadded_size < UNPADDED_SIZE_MIN || unpadded_size > UNPADDED_SIZE_MAX || uncompressed_size > LZMA_VLI_MAX) return LZMA_PROG_ERROR; - s = (index_stream *)(i->streams.rightmost); - g = (index_group *)(s->groups.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); + index_group *g = (index_group *)(s->groups.rightmost); - compressed_base = g == NULL ? 0 + const lzma_vli compressed_base = g == NULL ? 0 : vli_ceil4(g->records[g->last].unpadded_sum); - uncompressed_base = g == NULL ? 0 + const lzma_vli uncompressed_base = g == NULL ? 0 : g->records[g->last].uncompressed_sum; - index_list_size_add = lzma_vli_size(unpadded_size) + const uint32_t index_list_size_add = lzma_vli_size(unpadded_size) + lzma_vli_size(uncompressed_size); // Check that the file size will stay within limits. @@ -780,10 +764,9 @@ index_cat_helper(const index_cat_info *info, index_stream *this) extern LZMA_API(lzma_ret) -lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, - lzma_allocator *allocator) +lzma_index_cat(lzma_index *restrict dest, lzma_index *restrict src, + const lzma_allocator *allocator) { - index_cat_info info; const lzma_vli dest_file_size = lzma_index_file_size(dest); // Check that we don't exceed the file size limits. @@ -813,12 +796,10 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, index_stream *s = (index_stream *)(dest->streams.rightmost); index_group *g = (index_group *)(s->groups.rightmost); if (g != NULL && g->last + 1 < g->allocated) { - index_group *newg; - assert(g->node.left == NULL); assert(g->node.right == NULL); - newg = lzma_alloc(sizeof(index_group) + index_group *newg = lzma_alloc(sizeof(index_group) + (g->last + 1) * sizeof(index_record), allocator); @@ -848,17 +829,21 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, s->groups.rightmost = &newg->node; lzma_free(g, allocator); + + // NOTE: newg isn't leaked here because + // newg == (void *)&newg->node. } } // Add all the Streams from src to dest. Update the base offsets // of each Stream from src. - info.uncompressed_size = dest->uncompressed_size; - info.file_size = dest_file_size; - info.stream_number_add = dest->streams.count; - info.block_number_add = dest->record_count; - info.streams = &dest->streams; - + const index_cat_info info = { + .uncompressed_size = dest->uncompressed_size, + .file_size = dest_file_size, + .stream_number_add = dest->streams.count, + .block_number_add = dest->record_count, + .streams = &dest->streams, + }; index_cat_helper(&info, (index_stream *)(src->streams.root)); // Update info about all the combined Streams. @@ -877,26 +862,18 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, /// Duplicate an index_stream. static index_stream * -index_dup_stream(const index_stream *src, lzma_allocator *allocator) +index_dup_stream(const index_stream *src, const lzma_allocator *allocator) { - index_stream *dest; - index_group *destg; - index_group *srcg; - size_t i = 0; - // Catch a somewhat theoretical integer overflow. if (src->record_count > PREALLOC_MAX) return NULL; // Allocate and initialize a new Stream. - dest = index_stream_init(src->node.compressed_base, + index_stream *dest = index_stream_init(src->node.compressed_base, src->node.uncompressed_base, src->number, src->block_number_base, allocator); - - // Return immediately if allocation failed or if there are - // no groups to duplicate. - if (dest == NULL || src->groups.leftmost == NULL) - return dest; + if (dest == NULL) + return NULL; // Copy the overall information. dest->record_count = src->record_count; @@ -904,10 +881,14 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) dest->stream_flags = src->stream_flags; dest->stream_padding = src->stream_padding; + // Return if there are no groups to duplicate. + if (src->groups.leftmost == NULL) + return dest; + // Allocate memory for the Records. We put all the Records into // a single group. It's simplest and also tends to make // lzma_index_locate() a little bit faster with very big Indexes. - destg = lzma_alloc(sizeof(index_group) + index_group *destg = lzma_alloc(sizeof(index_group) + src->record_count * sizeof(index_record), allocator); if (destg == NULL) { @@ -923,7 +904,8 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) destg->last = src->record_count - 1; // Go through all the groups in src and copy the Records into destg. - srcg = (index_group *)(src->groups.leftmost); + const index_group *srcg = (const index_group *)(src->groups.leftmost); + size_t i = 0; do { memcpy(destg->records + i, srcg->records, (srcg->last + 1) * sizeof(index_record)); @@ -941,11 +923,8 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) extern LZMA_API(lzma_index *) -lzma_index_dup(const lzma_index *src, lzma_allocator *allocator) +lzma_index_dup(const lzma_index *src, const lzma_allocator *allocator) { - index_stream *srcstream; - index_stream *deststream; - // Allocate the base structure (no initial Stream). lzma_index *dest = index_init_plain(allocator); if (dest == NULL) @@ -958,9 +937,11 @@ lzma_index_dup(const lzma_index *src, lzma_allocator *allocator) dest->index_list_size = src->index_list_size; // Copy the Streams and the groups in them. - srcstream = (index_stream *)(src->streams.leftmost); + const index_stream *srcstream + = (const index_stream *)(src->streams.leftmost); do { - deststream = index_dup_stream(srcstream, allocator); + index_stream *deststream = index_dup_stream( + srcstream, allocator); if (deststream == NULL) { lzma_index_end(dest, allocator); return NULL; @@ -1031,6 +1012,8 @@ iter_set_info(lzma_index_iter *iter) iter->internal[ITER_GROUP].p = NULL; } + // NOTE: lzma_index_iter.stream.number is lzma_vli but we use uint32_t + // internally. iter->stream.number = stream->number; iter->stream.block_count = stream->record_count; iter->stream.compressed_offset = stream->node.compressed_base; @@ -1119,19 +1102,14 @@ lzma_index_iter_rewind(lzma_index_iter *iter) extern LZMA_API(lzma_bool) lzma_index_iter_next(lzma_index_iter *iter, lzma_index_iter_mode mode) { - const lzma_index *i; - const index_stream *stream; - const index_group *group; - size_t record; - // Catch unsupported mode values. if ((unsigned int)(mode) > LZMA_INDEX_ITER_NONEMPTY_BLOCK) return true; - i = iter->internal[ITER_INDEX].p; - stream = iter->internal[ITER_STREAM].p; - group = NULL; - record = iter->internal[ITER_RECORD].s; + const lzma_index *i = iter->internal[ITER_INDEX].p; + const index_stream *stream = iter->internal[ITER_STREAM].p; + const index_group *group = NULL; + size_t record = iter->internal[ITER_RECORD].s; // If we are being asked for the next Stream, leave group to NULL // so that the rest of the this function thinks that this Stream @@ -1231,10 +1209,6 @@ again: extern LZMA_API(lzma_bool) lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) { - const index_stream *stream; - const index_group *group; - size_t left, right; - const lzma_index *i = iter->internal[ITER_INDEX].p; // If the target is past the end of the file, return immediately. @@ -1242,12 +1216,12 @@ lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) return true; // Locate the Stream containing the target offset. - stream = index_tree_locate(&i->streams, target); + const index_stream *stream = index_tree_locate(&i->streams, target); assert(stream != NULL); target -= stream->node.uncompressed_base; // Locate the group containing the target offset. - group = index_tree_locate(&stream->groups, target); + const index_group *group = index_tree_locate(&stream->groups, target); assert(group != NULL); // Use binary search to locate the exact Record. It is the first @@ -1255,8 +1229,8 @@ lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) // This is because we want the rightmost Record that fullfills the // search criterion. It is possible that there are empty Blocks; // we don't want to return them. - left = 0; - right = group->last; + size_t left = 0; + size_t right = group->last; while (left < right) { const size_t pos = left + (right - left) / 2; diff --git a/Utilities/cmliblzma/liblzma/common/index_decoder.c b/Utilities/cmliblzma/liblzma/common/index_decoder.c index 943cfd5..cc07a1b 100644 --- a/Utilities/cmliblzma/liblzma/common/index_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/index_decoder.c @@ -14,7 +14,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INDICATOR, SEQ_COUNT, @@ -50,18 +50,20 @@ struct lzma_coder_s { /// CRC32 of the List of Records field uint32_t crc32; -}; +} lzma_index_coder; static lzma_ret -index_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +index_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, - uint8_t *LZMA_RESTRICT out lzma_attribute((__unused__)), - size_t *LZMA_RESTRICT out_pos lzma_attribute((__unused__)), + uint8_t *restrict out lzma_attribute((__unused__)), + size_t *restrict out_pos lzma_attribute((__unused__)), size_t out_size lzma_attribute((__unused__)), lzma_action action lzma_attribute((__unused__))) { + lzma_index_coder *coder = coder_ptr; + // Similar optimization as in index_encoder.c const size_t in_start = *in_pos; lzma_ret ret = LZMA_OK; @@ -207,8 +209,9 @@ out: static void -index_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +index_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_index_coder *coder = coder_ptr; lzma_index_end(coder->index, allocator); lzma_free(coder, allocator); return; @@ -216,9 +219,11 @@ index_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +index_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_index_coder *coder = coder_ptr; + *memusage = lzma_index_memusage(1, coder->count); *old_memlimit = coder->memlimit; @@ -234,7 +239,7 @@ index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, static lzma_ret -index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, +index_decoder_reset(lzma_index_coder *coder, const lzma_allocator *allocator, lzma_index **i, uint64_t memlimit) { // Remember the pointer given by the application. We will set it @@ -251,7 +256,7 @@ index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, // Initialize the rest. coder->sequence = SEQ_INDICATOR; - coder->memlimit = memlimit; + coder->memlimit = my_max(1, memlimit); coder->count = 0; // Needs to be initialized due to _memconfig(). coder->pos = 0; coder->crc32 = 0; @@ -261,35 +266,37 @@ index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, static lzma_ret -index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_index **i, uint64_t memlimit) { lzma_next_coder_init(&index_decoder_init, next, allocator); - if (i == NULL || memlimit == 0) + if (i == NULL) return LZMA_PROG_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_index_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_index_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &index_decode; next->end = &index_decoder_end; next->memconfig = &index_decoder_memconfig; - next->coder->index = NULL; + coder->index = NULL; } else { - lzma_index_end(next->coder->index, allocator); + lzma_index_end(coder->index, allocator); } - return index_decoder_reset(next->coder, allocator, i, memlimit); + return index_decoder_reset(coder, allocator, i, memlimit); } extern LZMA_API(lzma_ret) lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) { - lzma_next_strm_init2(index_decoder_init, strm, i, memlimit); + lzma_next_strm_init(index_decoder_init, strm, i, memlimit); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -299,27 +306,25 @@ lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) extern LZMA_API(lzma_ret) -lzma_index_buffer_decode( - lzma_index **i, uint64_t *memlimit, lzma_allocator *allocator, +lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { - lzma_coder coder; - lzma_ret ret; - - // Store the input start position so that we can restore it in case - // of an error. - const size_t in_start = *in_pos; - // Sanity checks if (i == NULL || memlimit == NULL || in == NULL || in_pos == NULL || *in_pos > in_size) return LZMA_PROG_ERROR; // Initialize the decoder. + lzma_index_coder coder; return_if_error(index_decoder_reset(&coder, allocator, i, *memlimit)); + // Store the input start position so that we can restore it in case + // of an error. + const size_t in_start = *in_pos; + // Do the actual decoding. - ret = index_decode(&coder, allocator, in, in_pos, in_size, + lzma_ret ret = index_decode(&coder, allocator, in, in_pos, in_size, NULL, NULL, 0, LZMA_RUN); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/index_encoder.c b/Utilities/cmliblzma/liblzma/common/index_encoder.c index 194bf21..ac97d0c 100644 --- a/Utilities/cmliblzma/liblzma/common/index_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/index_encoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INDICATOR, SEQ_COUNT, @@ -37,19 +37,21 @@ struct lzma_coder_s { /// CRC32 of the List of Records field uint32_t crc32; -}; +} lzma_index_coder; static lzma_ret -index_encode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in lzma_attribute((__unused__)), - size_t *LZMA_RESTRICT in_pos lzma_attribute((__unused__)), +index_encode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in lzma_attribute((__unused__)), + size_t *restrict in_pos lzma_attribute((__unused__)), size_t in_size lzma_attribute((__unused__)), - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_action action lzma_attribute((__unused__))) { + lzma_index_coder *coder = coder_ptr; + // Position where to start calculating CRC32. The idea is that we // need to call lzma_crc32() only once per call to index_encode(). const size_t out_start = *out_pos; @@ -159,7 +161,7 @@ out: static void -index_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +index_encoder_end(void *coder, const lzma_allocator *allocator) { lzma_free(coder, allocator); return; @@ -167,7 +169,7 @@ index_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static void -index_encoder_reset(lzma_coder *coder, const lzma_index *i) +index_encoder_reset(lzma_index_coder *coder, const lzma_index *i) { lzma_index_iter_init(&coder->iter, i); @@ -181,7 +183,7 @@ index_encoder_reset(lzma_coder *coder, const lzma_index *i) extern lzma_ret -lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_index_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_index *i) { lzma_next_coder_init(&lzma_index_encoder_init, next, allocator); @@ -190,7 +192,7 @@ lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_PROG_ERROR; if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + next->coder = lzma_alloc(sizeof(lzma_index_coder), allocator); if (next->coder == NULL) return LZMA_MEM_ERROR; @@ -207,7 +209,7 @@ lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_index_encoder(lzma_stream *strm, const lzma_index *i) { - lzma_next_strm_init1(lzma_index_encoder_init, strm, i); + lzma_next_strm_init(lzma_index_encoder_init, strm, i); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -220,10 +222,6 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_coder coder; - size_t out_start; - lzma_ret ret; - // Validate the arguments. if (i == NULL || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; @@ -234,12 +232,13 @@ lzma_index_buffer_encode(const lzma_index *i, // The Index encoder needs just one small data structure so we can // allocate it on stack. + lzma_index_coder coder; index_encoder_reset(&coder, i); // Do the actual encoding. This should never fail, but store // the original *out_pos just in case. - out_start = *out_pos; - ret = index_encode(&coder, NULL, NULL, NULL, 0, + const size_t out_start = *out_pos; + lzma_ret ret = index_encode(&coder, NULL, NULL, NULL, 0, out, out_pos, out_size, LZMA_RUN); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/index_encoder.h b/Utilities/cmliblzma/liblzma/common/index_encoder.h index a13c94d..4d55cd1 100644 --- a/Utilities/cmliblzma/liblzma/common/index_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/index_encoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_index_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_index *i); + const lzma_allocator *allocator, const lzma_index *i); #endif diff --git a/Utilities/cmliblzma/liblzma/common/index_hash.c b/Utilities/cmliblzma/liblzma/common/index_hash.c index 0cf86b3..d7a0344 100644 --- a/Utilities/cmliblzma/liblzma/common/index_hash.c +++ b/Utilities/cmliblzma/liblzma/common/index_hash.c @@ -70,7 +70,8 @@ struct lzma_index_hash_s { extern LZMA_API(lzma_index_hash *) -lzma_index_hash_init(lzma_index_hash *index_hash, lzma_allocator *allocator) +lzma_index_hash_init(lzma_index_hash *index_hash, + const lzma_allocator *allocator) { if (index_hash == NULL) { index_hash = lzma_alloc(sizeof(lzma_index_hash), allocator); @@ -101,7 +102,8 @@ lzma_index_hash_init(lzma_index_hash *index_hash, lzma_allocator *allocator) extern LZMA_API(void) -lzma_index_hash_end(lzma_index_hash *index_hash, lzma_allocator *allocator) +lzma_index_hash_end(lzma_index_hash *index_hash, + const lzma_allocator *allocator) { lzma_free(index_hash, allocator); return; @@ -124,14 +126,13 @@ static lzma_ret hash_append(lzma_index_hash_info *info, lzma_vli unpadded_size, lzma_vli uncompressed_size) { - const lzma_vli sizes[2] = { unpadded_size, uncompressed_size }; - info->blocks_size += vli_ceil4(unpadded_size); info->uncompressed_size += uncompressed_size; info->index_list_size += lzma_vli_size(unpadded_size) + lzma_vli_size(uncompressed_size); ++info->count; + const lzma_vli sizes[2] = { unpadded_size, uncompressed_size }; lzma_check_update(&info->check, LZMA_CHECK_BEST, (const uint8_t *)(sizes), sizeof(sizes)); @@ -174,9 +175,6 @@ extern LZMA_API(lzma_ret) lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, size_t *in_pos, size_t in_size) { - size_t in_start; - lzma_ret ret; - // Catch zero input buffer here, because in contrast to Index encoder // and decoder functions, applications call this function directly // instead of via lzma_code(), which does the buffer checking. @@ -186,8 +184,8 @@ lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, // NOTE: This function has many similarities to index_encode() and // index_decode() functions found from index_encoder.c and // index_decoder.c. See the comments especially in index_encoder.c. - in_start = *in_pos; - ret = LZMA_OK; + const size_t in_start = *in_pos; + lzma_ret ret = LZMA_OK; while (*in_pos < in_size) switch (index_hash->sequence) { diff --git a/Utilities/cmliblzma/liblzma/common/memcmplen.h b/Utilities/cmliblzma/liblzma/common/memcmplen.h new file mode 100644 index 0000000..c1efc9e --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/memcmplen.h @@ -0,0 +1,175 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file memcmplen.h +/// \brief Optimized comparison of two buffers +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_MEMCMPLEN_H +#define LZMA_MEMCMPLEN_H + +#include "common.h" + +#ifdef HAVE_IMMINTRIN_H +# include <immintrin.h> +#endif + + +/// Find out how many equal bytes the two buffers have. +/// +/// \param buf1 First buffer +/// \param buf2 Second buffer +/// \param len How many bytes have already been compared and will +/// be assumed to match +/// \param limit How many bytes to compare at most, including the +/// already-compared bytes. This must be significantly +/// smaller than UINT32_MAX to avoid integer overflows. +/// Up to LZMA_MEMCMPLEN_EXTRA bytes may be read past +/// the specified limit from both buf1 and buf2. +/// +/// \return Number of equal bytes in the buffers is returned. +/// This is always at least len and at most limit. +/// +/// \note LZMA_MEMCMPLEN_EXTRA defines how many extra bytes may be read. +/// It's rounded up to 2^n. This extra amount needs to be +/// allocated in the buffers being used. It needs to be +/// initialized too to keep Valgrind quiet. +static inline uint32_t lzma_attribute((__always_inline__)) +lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2, + uint32_t len, uint32_t limit) +{ + assert(len <= limit); + assert(limit <= UINT32_MAX / 2); + +#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ + && ((TUKLIB_GNUC_REQ(3, 4) && defined(__x86_64__)) \ + || (defined(__INTEL_COMPILER) && defined(__x86_64__)) \ + || (defined(__INTEL_COMPILER) && defined(_M_X64)) \ + || (defined(_MSC_VER) && defined(_M_X64))) + // NOTE: This will use 64-bit unaligned access which + // TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit, but + // it's convenient here at least as long as it's x86-64 only. + // + // I keep this x86-64 only for now since that's where I know this + // to be a good method. This may be fine on other 64-bit CPUs too. + // On big endian one should use xor instead of subtraction and switch + // to __builtin_clzll(). +#define LZMA_MEMCMPLEN_EXTRA 8 + while (len < limit) { + const uint64_t x = *(const uint64_t *)(buf1 + len) + - *(const uint64_t *)(buf2 + len); + if (x != 0) { +# if defined(_M_X64) // MSVC or Intel C compiler on Windows + unsigned long tmp; + _BitScanForward64(&tmp, x); + len += (uint32_t)tmp >> 3; +# else // GCC, clang, or Intel C compiler + len += (uint32_t)__builtin_ctzll(x) >> 3; +# endif + return my_min(len, limit); + } + + len += 8; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ + && defined(HAVE__MM_MOVEMASK_EPI8) \ + && ((defined(__GNUC__) && defined(__SSE2_MATH__)) \ + || (defined(__INTEL_COMPILER) && defined(__SSE2__)) \ + || (defined(_MSC_VER) && defined(_M_IX86_FP) \ + && _M_IX86_FP >= 2)) + // NOTE: Like above, this will use 128-bit unaligned access which + // TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit. + // + // SSE2 version for 32-bit and 64-bit x86. On x86-64 the above + // version is sometimes significantly faster and sometimes + // slightly slower than this SSE2 version, so this SSE2 + // version isn't used on x86-64. +# define LZMA_MEMCMPLEN_EXTRA 16 + while (len < limit) { + const uint32_t x = 0xFFFF ^ _mm_movemask_epi8(_mm_cmpeq_epi8( + _mm_loadu_si128((const __m128i *)(buf1 + len)), + _mm_loadu_si128((const __m128i *)(buf2 + len)))); + + if (x != 0) { +# if defined(__INTEL_COMPILER) + len += _bit_scan_forward(x); +# elif defined(_MSC_VER) + unsigned long tmp; + _BitScanForward(&tmp, x); + len += tmp; +# else + len += __builtin_ctz(x); +# endif + return my_min(len, limit); + } + + len += 16; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && !defined(WORDS_BIGENDIAN) + // Generic 32-bit little endian method +# define LZMA_MEMCMPLEN_EXTRA 4 + while (len < limit) { + uint32_t x = *(const uint32_t *)(buf1 + len) + - *(const uint32_t *)(buf2 + len); + if (x != 0) { + if ((x & 0xFFFF) == 0) { + len += 2; + x >>= 16; + } + + if ((x & 0xFF) == 0) + ++len; + + return my_min(len, limit); + } + + len += 4; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && defined(WORDS_BIGENDIAN) + // Generic 32-bit big endian method +# define LZMA_MEMCMPLEN_EXTRA 4 + while (len < limit) { + uint32_t x = *(const uint32_t *)(buf1 + len) + ^ *(const uint32_t *)(buf2 + len); + if (x != 0) { + if ((x & 0xFFFF0000) == 0) { + len += 2; + x <<= 16; + } + + if ((x & 0xFF000000) == 0) + ++len; + + return my_min(len, limit); + } + + len += 4; + } + + return limit; + +#else + // Simple portable version that doesn't use unaligned access. +# define LZMA_MEMCMPLEN_EXTRA 0 + while (len < limit && buf1[len] == buf2[len]) + ++len; + + return len; +#endif +} + +#endif diff --git a/Utilities/cmliblzma/liblzma/common/outqueue.c b/Utilities/cmliblzma/liblzma/common/outqueue.c new file mode 100644 index 0000000..2dc8a38 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/outqueue.c @@ -0,0 +1,184 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file outqueue.c +/// \brief Output queue handling in multithreaded coding +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "outqueue.h" + + +/// This is to ease integer overflow checking: We may allocate up to +/// 2 * LZMA_THREADS_MAX buffers and we need some extra memory for other +/// data structures (that's the second /2). +#define BUF_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2) + + +static lzma_ret +get_options(uint64_t *bufs_alloc_size, uint32_t *bufs_count, + uint64_t buf_size_max, uint32_t threads) +{ + if (threads > LZMA_THREADS_MAX || buf_size_max > BUF_SIZE_MAX) + return LZMA_OPTIONS_ERROR; + + // The number of buffers is twice the number of threads. + // This wastes RAM but keeps the threads busy when buffers + // finish out of order. + // + // NOTE: If this is changed, update BUF_SIZE_MAX too. + *bufs_count = threads * 2; + *bufs_alloc_size = *bufs_count * buf_size_max; + + return LZMA_OK; +} + + +extern uint64_t +lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads) +{ + uint64_t bufs_alloc_size; + uint32_t bufs_count; + + if (get_options(&bufs_alloc_size, &bufs_count, buf_size_max, threads) + != LZMA_OK) + return UINT64_MAX; + + return sizeof(lzma_outq) + bufs_count * sizeof(lzma_outbuf) + + bufs_alloc_size; +} + + +extern lzma_ret +lzma_outq_init(lzma_outq *outq, const lzma_allocator *allocator, + uint64_t buf_size_max, uint32_t threads) +{ + uint64_t bufs_alloc_size; + uint32_t bufs_count; + + // Set bufs_count and bufs_alloc_size. + return_if_error(get_options(&bufs_alloc_size, &bufs_count, + buf_size_max, threads)); + + // Allocate memory if needed. + if (outq->buf_size_max != buf_size_max + || outq->bufs_allocated != bufs_count) { + lzma_outq_end(outq, allocator); + +#if SIZE_MAX < UINT64_MAX + if (bufs_alloc_size > SIZE_MAX) + return LZMA_MEM_ERROR; +#endif + + outq->bufs = lzma_alloc(bufs_count * sizeof(lzma_outbuf), + allocator); + outq->bufs_mem = lzma_alloc((size_t)(bufs_alloc_size), + allocator); + + if (outq->bufs == NULL || outq->bufs_mem == NULL) { + lzma_outq_end(outq, allocator); + return LZMA_MEM_ERROR; + } + } + + // Initialize the rest of the main structure. Initialization of + // outq->bufs[] is done when they are actually needed. + outq->buf_size_max = (size_t)(buf_size_max); + outq->bufs_allocated = bufs_count; + outq->bufs_pos = 0; + outq->bufs_used = 0; + outq->read_pos = 0; + + return LZMA_OK; +} + + +extern void +lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator) +{ + lzma_free(outq->bufs, allocator); + outq->bufs = NULL; + + lzma_free(outq->bufs_mem, allocator); + outq->bufs_mem = NULL; + + return; +} + + +extern lzma_outbuf * +lzma_outq_get_buf(lzma_outq *outq) +{ + // Caller must have checked it with lzma_outq_has_buf(). + assert(outq->bufs_used < outq->bufs_allocated); + + // Initialize the new buffer. + lzma_outbuf *buf = &outq->bufs[outq->bufs_pos]; + buf->buf = outq->bufs_mem + outq->bufs_pos * outq->buf_size_max; + buf->size = 0; + buf->finished = false; + + // Update the queue state. + if (++outq->bufs_pos == outq->bufs_allocated) + outq->bufs_pos = 0; + + ++outq->bufs_used; + + return buf; +} + + +extern bool +lzma_outq_is_readable(const lzma_outq *outq) +{ + uint32_t i = outq->bufs_pos - outq->bufs_used; + if (outq->bufs_pos < outq->bufs_used) + i += outq->bufs_allocated; + + return outq->bufs[i].finished; +} + + +extern lzma_ret +lzma_outq_read(lzma_outq *restrict outq, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_vli *restrict unpadded_size, + lzma_vli *restrict uncompressed_size) +{ + // There must be at least one buffer from which to read. + if (outq->bufs_used == 0) + return LZMA_OK; + + // Get the buffer. + uint32_t i = outq->bufs_pos - outq->bufs_used; + if (outq->bufs_pos < outq->bufs_used) + i += outq->bufs_allocated; + + lzma_outbuf *buf = &outq->bufs[i]; + + // If it isn't finished yet, we cannot read from it. + if (!buf->finished) + return LZMA_OK; + + // Copy from the buffer to output. + lzma_bufcpy(buf->buf, &outq->read_pos, buf->size, + out, out_pos, out_size); + + // Return if we didn't get all the data from the buffer. + if (outq->read_pos < buf->size) + return LZMA_OK; + + // The buffer was finished. Tell the caller its size information. + *unpadded_size = buf->unpadded_size; + *uncompressed_size = buf->uncompressed_size; + + // Free this buffer for further use. + --outq->bufs_used; + outq->read_pos = 0; + + return LZMA_STREAM_END; +} diff --git a/Utilities/cmliblzma/liblzma/common/outqueue.h b/Utilities/cmliblzma/liblzma/common/outqueue.h new file mode 100644 index 0000000..079634d --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/outqueue.h @@ -0,0 +1,156 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file outqueue.h +/// \brief Output queue handling in multithreaded coding +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +/// Output buffer for a single thread +typedef struct { + /// Pointer to the output buffer of lzma_outq.buf_size_max bytes + uint8_t *buf; + + /// Amount of data written to buf + size_t size; + + /// Additional size information + lzma_vli unpadded_size; + lzma_vli uncompressed_size; + + /// True when no more data will be written into this buffer. + /// + /// \note This is read by another thread and thus access + /// to this variable needs a mutex. + bool finished; + +} lzma_outbuf; + + +typedef struct { + /// Array of buffers that are used cyclically. + lzma_outbuf *bufs; + + /// Memory allocated for all the buffers + uint8_t *bufs_mem; + + /// Amount of buffer space available in each buffer + size_t buf_size_max; + + /// Number of buffers allocated + uint32_t bufs_allocated; + + /// Position in the bufs array. The next buffer to be taken + /// into use is bufs[bufs_pos]. + uint32_t bufs_pos; + + /// Number of buffers in use + uint32_t bufs_used; + + /// Position in the buffer in lzma_outq_read() + size_t read_pos; + +} lzma_outq; + + +/** + * \brief Calculate the memory usage of an output queue + * + * \return Approximate memory usage in bytes or UINT64_MAX on error. + */ +extern uint64_t lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads); + + +/// \brief Initialize an output queue +/// +/// \param outq Pointer to an output queue. Before calling +/// this function the first time, *outq should +/// have been zeroed with memzero() so that this +/// function knows that there are no previous +/// allocations to free. +/// \param allocator Pointer to allocator or NULL +/// \param buf_size_max Maximum amount of data that a single buffer +/// in the queue may need to store. +/// \param threads Number of buffers that may be in use +/// concurrently. Note that more than this number +/// of buffers will actually get allocated to +/// improve performance when buffers finish +/// out of order. +/// +/// \return - LZMA_OK +/// - LZMA_MEM_ERROR +/// +extern lzma_ret lzma_outq_init( + lzma_outq *outq, const lzma_allocator *allocator, + uint64_t buf_size_max, uint32_t threads); + + +/// \brief Free the memory associated with the output queue +extern void lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator); + + +/// \brief Get a new buffer +/// +/// lzma_outq_has_buf() must be used to check that there is a buffer +/// available before calling lzma_outq_get_buf(). +/// +extern lzma_outbuf *lzma_outq_get_buf(lzma_outq *outq); + + +/// \brief Test if there is data ready to be read +/// +/// Call to this function must be protected with the same mutex that +/// is used to protect lzma_outbuf.finished. +/// +extern bool lzma_outq_is_readable(const lzma_outq *outq); + + +/// \brief Read finished data +/// +/// \param outq Pointer to an output queue +/// \param out Beginning of the output buffer +/// \param out_pos The next byte will be written to +/// out[*out_pos]. +/// \param out_size Size of the out buffer; the first byte into +/// which no data is written to is out[out_size]. +/// \param unpadded_size Unpadded Size from the Block encoder +/// \param uncompressed_size Uncompressed Size from the Block encoder +/// +/// \return - LZMA: All OK. Either no data was available or the buffer +/// being read didn't become empty yet. +/// - LZMA_STREAM_END: The buffer being read was finished. +/// *unpadded_size and *uncompressed_size were set. +/// +/// \note This reads lzma_outbuf.finished variables and thus call +/// to this function needs to be protected with a mutex. +/// +extern lzma_ret lzma_outq_read(lzma_outq *restrict outq, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, lzma_vli *restrict unpadded_size, + lzma_vli *restrict uncompressed_size); + + +/// \brief Test if there is at least one buffer free +/// +/// This must be used before getting a new buffer with lzma_outq_get_buf(). +/// +static inline bool +lzma_outq_has_buf(const lzma_outq *outq) +{ + return outq->bufs_used < outq->bufs_allocated; +} + + +/// \brief Test if the queue is completely empty +static inline bool +lzma_outq_is_empty(const lzma_outq *outq) +{ + return outq->bufs_used == 0; +} diff --git a/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c index 9e2e1da..b9745b5 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c @@ -15,13 +15,10 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags, - lzma_allocator *allocator, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder stream_decoder = LZMA_NEXT_CODER_INIT; - lzma_ret ret; - // Sanity checks if (in_pos == NULL || (in == NULL && *in_pos != in_size) || *in_pos > in_size || out_pos == NULL @@ -36,7 +33,8 @@ lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags, // Initialize the Stream decoder. // TODO: We need something to tell the decoder that it can use the // output buffer as workspace, and thus save significant amount of RAM. - ret = lzma_stream_decoder_init( + lzma_next_coder stream_decoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_stream_decoder_init( &stream_decoder, allocator, *memlimit, flags); if (ret == LZMA_OK) { diff --git a/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c index 8bca87f..af49554 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c @@ -42,13 +42,10 @@ lzma_stream_buffer_bound(size_t uncompressed_size) extern LZMA_API(lzma_ret) lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos_ptr, size_t out_size) { - lzma_stream_flags stream_flags = { 0 }; - lzma_block block = { 0 }; - size_t out_pos; - // Sanity checks if (filters == NULL || (unsigned int)(check) > LZMA_CHECK_ID_MAX || (in == NULL && in_size != 0) || out == NULL @@ -65,7 +62,7 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, // Use a local copy. We update *out_pos_ptr only if everything // succeeds. - out_pos = *out_pos_ptr; + size_t out_pos = *out_pos_ptr; // Check that there's enough space for both Stream Header and // Stream Footer. @@ -77,7 +74,10 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, out_size -= LZMA_STREAM_HEADER_SIZE; // Encode the Stream Header. - stream_flags.check = check; + lzma_stream_flags stream_flags = { + .version = 0, + .check = check, + }; if (lzma_stream_header_encode(&stream_flags, out + out_pos) != LZMA_OK) @@ -86,8 +86,11 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, out_pos += LZMA_STREAM_HEADER_SIZE; // Encode a Block but only if there is at least one byte of input. - block.check = check; - block.filters = filters; + lzma_block block = { + .version = 0, + .check = check, + .filters = filters, + }; if (in_size > 0) return_if_error(lzma_block_buffer_encode(&block, allocator, @@ -95,8 +98,6 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, // Index { - lzma_ret ret; - // Create an Index. It will have one Record if there was // at least one byte of input to encode. Otherwise the // Index will be empty. @@ -104,7 +105,7 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, if (i == NULL) return LZMA_MEM_ERROR; - ret = LZMA_OK; + lzma_ret ret = LZMA_OK; if (in_size > 0) ret = lzma_index_append(i, allocator, diff --git a/Utilities/cmliblzma/liblzma/common/stream_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_decoder.c index 5e9a220..fdd8ff2 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_decoder.c @@ -14,7 +14,7 @@ #include "block_decoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_STREAM_HEADER, SEQ_BLOCK_HEADER, @@ -57,6 +57,10 @@ struct lzma_coder_s { /// If true, LZMA_GET_CHECK is returned after decoding Stream Header. bool tell_any_check; + /// If true, we will tell the Block decoder to skip calculating + /// and verifying the integrity check. + bool ignore_check; + /// If true, we will decode concatenated Streams that possibly have /// Stream Padding between or after them. LZMA_STREAM_END is returned /// once the application isn't giving us any new input, and we aren't @@ -76,11 +80,11 @@ struct lzma_coder_s { /// Buffer to hold Stream Header, Block Header, and Stream Footer. /// Block Header has biggest maximum size. uint8_t buffer[LZMA_BLOCK_HEADER_SIZE_MAX]; -}; +} lzma_stream_coder; static lzma_ret -stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator) +stream_decoder_reset(lzma_stream_coder *coder, const lzma_allocator *allocator) { // Initialize the Index hash used to verify the Index. coder->index_hash = lzma_index_hash_init(coder->index_hash, allocator); @@ -96,18 +100,18 @@ stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +stream_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_stream_coder *coder = coder_ptr; + // When decoding the actual Block, it may be able to produce more // output even if we don't give it any new input. while (true) switch (coder->sequence) { case SEQ_STREAM_HEADER: { - lzma_ret ret; - // Copy the Stream Header to the internal buffer. lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); @@ -119,7 +123,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, coder->pos = 0; // Decode the Stream Header. - ret = lzma_stream_header_decode( + const lzma_ret ret = lzma_stream_header_decode( &coder->stream_flags, coder->buffer); if (ret != LZMA_OK) return ret == LZMA_FORMAT_ERROR && !coder->first_stream @@ -156,11 +160,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_BLOCK_HEADER: { - lzma_filter filters[LZMA_FILTERS_MAX + 1]; - uint64_t memusage; - lzma_ret ret; - size_t i; - if (*in_pos >= in_size) return LZMA_OK; @@ -189,20 +188,28 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, coder->pos = 0; - // Version 0 is currently the only possible version. - coder->block_options.version = 0; + // Version 1 is needed to support the .ignore_check option. + coder->block_options.version = 1; // Set up a buffer to hold the filter chain. Block Header // decoder will initialize all members of this array so // we don't need to do it here. + lzma_filter filters[LZMA_FILTERS_MAX + 1]; coder->block_options.filters = filters; // Decode the Block Header. return_if_error(lzma_block_header_decode(&coder->block_options, allocator, coder->buffer)); + // If LZMA_IGNORE_CHECK was used, this flag needs to be set. + // It has to be set after lzma_block_header_decode() because + // it always resets this to false. + coder->block_options.ignore_check = coder->ignore_check; + // Check the memory usage limit. - memusage = lzma_raw_decoder_memusage(filters); + const uint64_t memusage = lzma_raw_decoder_memusage(filters); + lzma_ret ret; + if (memusage == UINT64_MAX) { // One or more unknown Filter IDs. ret = LZMA_OPTIONS_ERROR; @@ -228,7 +235,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Free the allocated filter options since they are needed // only to initialize the Block decoder. - for (i = 0; i < LZMA_FILTERS_MAX; ++i) + for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) lzma_free(filters[i].options, allocator); coder->block_options.filters = NULL; @@ -264,8 +271,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_INDEX: { - lzma_ret ret; - // If we don't have any input, don't call // lzma_index_hash_decode() since it would return // LZMA_BUF_ERROR, which we must not do here. @@ -274,7 +279,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Decode the Index and compare it to the hash calculated // from the sizes of the Blocks (if any). - ret = lzma_index_hash_decode(coder->index_hash, + const lzma_ret ret = lzma_index_hash_decode(coder->index_hash, in, in_pos, in_size); if (ret != LZMA_STREAM_END) return ret; @@ -285,9 +290,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_STREAM_FOOTER: { - lzma_stream_flags footer_flags; - lzma_ret ret; - // Copy the Stream Footer to the internal buffer. lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); @@ -301,7 +303,8 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Decode the Stream Footer. The decoder gives // LZMA_FORMAT_ERROR if the magic bytes don't match, // so convert that return code to LZMA_DATA_ERROR. - ret = lzma_stream_footer_decode( + lzma_stream_flags footer_flags; + const lzma_ret ret = lzma_stream_footer_decode( &footer_flags, coder->buffer); if (ret != LZMA_OK) return ret == LZMA_FORMAT_ERROR @@ -374,8 +377,9 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, static void -stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +stream_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_stream_coder *coder = coder_ptr; lzma_next_end(&coder->block_decoder, allocator); lzma_index_hash_end(coder->index_hash, allocator); lzma_free(coder, allocator); @@ -384,16 +388,19 @@ stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_check -stream_decoder_get_check(const lzma_coder *coder) +stream_decoder_get_check(const void *coder_ptr) { + const lzma_stream_coder *coder = coder_ptr; return coder->stream_flags.check; } static lzma_ret -stream_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +stream_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_stream_coder *coder = coder_ptr; + *memusage = coder->memusage; *old_memlimit = coder->memlimit; @@ -409,48 +416,49 @@ stream_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, extern lzma_ret -lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_stream_decoder_init( + lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, uint32_t flags) { lzma_next_coder_init(&lzma_stream_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; - if (flags & ~LZMA_SUPPORTED_FLAGS) return LZMA_OPTIONS_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_stream_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &stream_decode; next->end = &stream_decoder_end; next->get_check = &stream_decoder_get_check; next->memconfig = &stream_decoder_memconfig; - next->coder->block_decoder = LZMA_NEXT_CODER_INIT; - next->coder->index_hash = NULL; + coder->block_decoder = LZMA_NEXT_CODER_INIT; + coder->index_hash = NULL; } - next->coder->memlimit = memlimit; - next->coder->memusage = LZMA_MEMUSAGE_BASE; - next->coder->tell_no_check = (flags & LZMA_TELL_NO_CHECK) != 0; - next->coder->tell_unsupported_check + coder->memlimit = my_max(1, memlimit); + coder->memusage = LZMA_MEMUSAGE_BASE; + coder->tell_no_check = (flags & LZMA_TELL_NO_CHECK) != 0; + coder->tell_unsupported_check = (flags & LZMA_TELL_UNSUPPORTED_CHECK) != 0; - next->coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0; - next->coder->concatenated = (flags & LZMA_CONCATENATED) != 0; - next->coder->first_stream = true; + coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0; + coder->ignore_check = (flags & LZMA_IGNORE_CHECK) != 0; + coder->concatenated = (flags & LZMA_CONCATENATED) != 0; + coder->first_stream = true; - return stream_decoder_reset(next->coder, allocator); + return stream_decoder_reset(coder, allocator); } extern LZMA_API(lzma_ret) lzma_stream_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init2(lzma_stream_decoder_init, strm, memlimit, flags); + lzma_next_strm_init(lzma_stream_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/stream_decoder.h b/Utilities/cmliblzma/liblzma/common/stream_decoder.h index e54ac28..c13c6ba 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/stream_decoder.h @@ -15,7 +15,8 @@ #include "common.h" -extern lzma_ret lzma_stream_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, uint64_t memlimit, uint32_t flags); +extern lzma_ret lzma_stream_decoder_init( + lzma_next_coder *next, const lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags); #endif diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_encoder.c index 1ba45ac..858cba4 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_encoder.c @@ -10,12 +10,11 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_encoder.h" #include "block_encoder.h" #include "index_encoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_STREAM_HEADER, SEQ_BLOCK_INIT, @@ -26,7 +25,7 @@ struct lzma_coder_s { } sequence; /// True if Block encoder has been initialized by - /// lzma_stream_encoder_init() or stream_encoder_update() + /// stream_encoder_init() or stream_encoder_update() /// and thus doesn't need to be initialized in stream_encode(). bool block_encoder_is_initialized; @@ -56,11 +55,11 @@ struct lzma_coder_s { /// Buffer to hold Stream Header, Block Header, and Stream Footer. /// Block Header has biggest maximum size. uint8_t buffer[LZMA_BLOCK_HEADER_SIZE_MAX]; -}; +} lzma_stream_coder; static lzma_ret -block_encoder_init(lzma_coder *coder, lzma_allocator *allocator) +block_encoder_init(lzma_stream_coder *coder, const lzma_allocator *allocator) { // Prepare the Block options. Even though Block encoder doesn't need // compressed_size, uncompressed_size, and header_size to be @@ -79,11 +78,13 @@ block_encoder_init(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +stream_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_stream_coder *coder = coder_ptr; + // Main loop while (*out_pos < out_size) switch (coder->sequence) { @@ -126,7 +127,7 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } // Initialize the Block encoder unless it was already - // initialized by lzma_stream_encoder_init() or + // initialized by stream_encoder_init() or // stream_encoder_update(). if (!coder->block_encoder_is_initialized) return_if_error(block_encoder_init(coder, allocator)); @@ -147,13 +148,12 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_BLOCK_ENCODE: { - lzma_vli unpadded_size; - - static const lzma_action convert[4] = { + static const lzma_action convert[LZMA_ACTION_MAX + 1] = { LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FINISH, LZMA_FINISH, + LZMA_FINISH, }; const lzma_ret ret = coder->block_encoder.code( @@ -164,7 +164,7 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, return ret; // Add a new Index Record. - unpadded_size = lzma_block_unpadded_size( + const lzma_vli unpadded_size = lzma_block_unpadded_size( &coder->block_options); assert(unpadded_size != 0); return_if_error(lzma_index_append(coder->index, allocator, @@ -176,12 +176,6 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_INDEX_ENCODE: { - const lzma_stream_flags stream_flags = { - 0, - lzma_index_size(coder->index), - coder->block_options.check, - }; - // Call the Index encoder. It doesn't take any input, so // those pointers can be NULL. const lzma_ret ret = coder->index_encoder.code( @@ -192,6 +186,11 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, return ret; // Encode the Stream Footer into coder->buffer. + const lzma_stream_flags stream_flags = { + .version = 0, + .backward_size = lzma_index_size(coder->index), + .check = coder->block_options.check, + }; if (lzma_stream_footer_encode(&stream_flags, coder->buffer) != LZMA_OK) @@ -212,15 +211,15 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, static void -stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +stream_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { - size_t i; + lzma_stream_coder *coder = coder_ptr; lzma_next_end(&coder->block_encoder, allocator); lzma_next_end(&coder->index_encoder, allocator); lzma_index_end(coder->index, allocator); - for (i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) lzma_free(coder->filters[i].options, allocator); lzma_free(coder, allocator); @@ -229,22 +228,20 @@ stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +stream_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters, const lzma_filter *reversed_filters) { - size_t i; + lzma_stream_coder *coder = coder_ptr; if (coder->sequence <= SEQ_BLOCK_INIT) { - lzma_ret ret; - // There is no incomplete Block waiting to be finished, // thus we can change the whole filter chain. Start by // trying to initialize the Block encoder with the new // chain. This way we detect if the chain is valid. coder->block_encoder_is_initialized = false; coder->block_options.filters = (lzma_filter *)(filters); - ret = block_encoder_init(coder, allocator); + const lzma_ret ret = block_encoder_init(coder, allocator); coder->block_options.filters = coder->filters; if (ret != LZMA_OK) return ret; @@ -264,62 +261,66 @@ stream_encoder_update(lzma_coder *coder, lzma_allocator *allocator, } // Free the copy of the old chain and make a copy of the new chain. - for (i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) lzma_free(coder->filters[i].options, allocator); return lzma_filters_copy(filters, coder->filters, allocator); } -extern lzma_ret -lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +static lzma_ret +stream_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters, lzma_check check) { - lzma_stream_flags stream_flags = { 0, 0, check }; - - lzma_next_coder_init(&lzma_stream_encoder_init, next, allocator); + lzma_next_coder_init(&stream_encoder_init, next, allocator); if (filters == NULL) return LZMA_PROG_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_stream_coder *coder = next->coder; + + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &stream_encode; next->end = &stream_encoder_end; next->update = &stream_encoder_update; - next->coder->filters[0].id = LZMA_VLI_UNKNOWN; - next->coder->block_encoder = LZMA_NEXT_CODER_INIT; - next->coder->index_encoder = LZMA_NEXT_CODER_INIT; - next->coder->index = NULL; + coder->filters[0].id = LZMA_VLI_UNKNOWN; + coder->block_encoder = LZMA_NEXT_CODER_INIT; + coder->index_encoder = LZMA_NEXT_CODER_INIT; + coder->index = NULL; } // Basic initializations - next->coder->sequence = SEQ_STREAM_HEADER; - next->coder->block_options.version = 0; - next->coder->block_options.check = check; + coder->sequence = SEQ_STREAM_HEADER; + coder->block_options.version = 0; + coder->block_options.check = check; // Initialize the Index - lzma_index_end(next->coder->index, allocator); - next->coder->index = lzma_index_init(allocator); - if (next->coder->index == NULL) + lzma_index_end(coder->index, allocator); + coder->index = lzma_index_init(allocator); + if (coder->index == NULL) return LZMA_MEM_ERROR; // Encode the Stream Header + lzma_stream_flags stream_flags = { + .version = 0, + .check = check, + }; return_if_error(lzma_stream_header_encode( - &stream_flags, next->coder->buffer)); + &stream_flags, coder->buffer)); - next->coder->buffer_pos = 0; - next->coder->buffer_size = LZMA_STREAM_HEADER_SIZE; + coder->buffer_pos = 0; + coder->buffer_size = LZMA_STREAM_HEADER_SIZE; // Initialize the Block encoder. This way we detect unsupported // filter chains when initializing the Stream encoder instead of // giving an error after Stream Header has already written out. - return stream_encoder_update( - next->coder, allocator, filters, NULL); + return stream_encoder_update(coder, allocator, filters, NULL); } @@ -327,11 +328,12 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder(lzma_stream *strm, const lzma_filter *filters, lzma_check check) { - lzma_next_strm_init2(lzma_stream_encoder_init, strm, filters, check); + lzma_next_strm_init(stream_encoder_init, strm, filters, check); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; strm->internal->supported_actions[LZMA_FULL_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_BARRIER] = true; strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c b/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c new file mode 100644 index 0000000..2efe44c --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c @@ -0,0 +1,1143 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_encoder_mt.c +/// \brief Multithreaded .xz Stream encoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_encoder.h" +#include "easy_preset.h" +#include "block_encoder.h" +#include "block_buffer_encoder.h" +#include "index_encoder.h" +#include "outqueue.h" + + +/// Maximum supported block size. This makes it simpler to prevent integer +/// overflows if we are given unusually large block size. +#define BLOCK_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX) + + +typedef enum { + /// Waiting for work. + THR_IDLE, + + /// Encoding is in progress. + THR_RUN, + + /// Encoding is in progress but no more input data will + /// be read. + THR_FINISH, + + /// The main thread wants the thread to stop whatever it was doing + /// but not exit. + THR_STOP, + + /// The main thread wants the thread to exit. We could use + /// cancellation but since there's stopped anyway, this is lazier. + THR_EXIT, + +} worker_state; + +typedef struct lzma_stream_coder_s lzma_stream_coder; + +typedef struct worker_thread_s worker_thread; +struct worker_thread_s { + worker_state state; + + /// Input buffer of coder->block_size bytes. The main thread will + /// put new input into this and update in_size accordingly. Once + /// no more input is coming, state will be set to THR_FINISH. + uint8_t *in; + + /// Amount of data available in the input buffer. This is modified + /// only by the main thread. + size_t in_size; + + /// Output buffer for this thread. This is set by the main + /// thread every time a new Block is started with this thread + /// structure. + lzma_outbuf *outbuf; + + /// Pointer to the main structure is needed when putting this + /// thread back to the stack of free threads. + lzma_stream_coder *coder; + + /// The allocator is set by the main thread. Since a copy of the + /// pointer is kept here, the application must not change the + /// allocator before calling lzma_end(). + const lzma_allocator *allocator; + + /// Amount of uncompressed data that has already been compressed. + uint64_t progress_in; + + /// Amount of compressed data that is ready. + uint64_t progress_out; + + /// Block encoder + lzma_next_coder block_encoder; + + /// Compression options for this Block + lzma_block block_options; + + /// Next structure in the stack of free worker threads. + worker_thread *next; + + mythread_mutex mutex; + mythread_cond cond; + + /// The ID of this thread is used to join the thread + /// when it's not needed anymore. + mythread thread_id; +}; + + +struct lzma_stream_coder_s { + enum { + SEQ_STREAM_HEADER, + SEQ_BLOCK, + SEQ_INDEX, + SEQ_STREAM_FOOTER, + } sequence; + + /// Start a new Block every block_size bytes of input unless + /// LZMA_FULL_FLUSH or LZMA_FULL_BARRIER is used earlier. + size_t block_size; + + /// The filter chain currently in use + lzma_filter filters[LZMA_FILTERS_MAX + 1]; + + + /// Index to hold sizes of the Blocks + lzma_index *index; + + /// Index encoder + lzma_next_coder index_encoder; + + + /// Stream Flags for encoding the Stream Header and Stream Footer. + lzma_stream_flags stream_flags; + + /// Buffer to hold Stream Header and Stream Footer. + uint8_t header[LZMA_STREAM_HEADER_SIZE]; + + /// Read position in header[] + size_t header_pos; + + + /// Output buffer queue for compressed data + lzma_outq outq; + + + /// Maximum wait time if cannot use all the input and cannot + /// fill the output buffer. This is in milliseconds. + uint32_t timeout; + + + /// Error code from a worker thread + lzma_ret thread_error; + + /// Array of allocated thread-specific structures + worker_thread *threads; + + /// Number of structures in "threads" above. This is also the + /// number of threads that will be created at maximum. + uint32_t threads_max; + + /// Number of thread structures that have been initialized, and + /// thus the number of worker threads actually created so far. + uint32_t threads_initialized; + + /// Stack of free threads. When a thread finishes, it puts itself + /// back into this stack. This starts as empty because threads + /// are created only when actually needed. + worker_thread *threads_free; + + /// The most recent worker thread to which the main thread writes + /// the new input from the application. + worker_thread *thr; + + + /// Amount of uncompressed data in Blocks that have already + /// been finished. + uint64_t progress_in; + + /// Amount of compressed data in Stream Header + Blocks that + /// have already been finished. + uint64_t progress_out; + + + mythread_mutex mutex; + mythread_cond cond; +}; + + +/// Tell the main thread that something has gone wrong. +static void +worker_error(worker_thread *thr, lzma_ret ret) +{ + assert(ret != LZMA_OK); + assert(ret != LZMA_STREAM_END); + + mythread_sync(thr->coder->mutex) { + if (thr->coder->thread_error == LZMA_OK) + thr->coder->thread_error = ret; + + mythread_cond_signal(&thr->coder->cond); + } + + return; +} + + +static worker_state +worker_encode(worker_thread *thr, worker_state state) +{ + assert(thr->progress_in == 0); + assert(thr->progress_out == 0); + + // Set the Block options. + thr->block_options = (lzma_block){ + .version = 0, + .check = thr->coder->stream_flags.check, + .compressed_size = thr->coder->outq.buf_size_max, + .uncompressed_size = thr->coder->block_size, + + // TODO: To allow changing the filter chain, the filters + // array must be copied to each worker_thread. + .filters = thr->coder->filters, + }; + + // Calculate maximum size of the Block Header. This amount is + // reserved in the beginning of the buffer so that Block Header + // along with Compressed Size and Uncompressed Size can be + // written there. + lzma_ret ret = lzma_block_header_size(&thr->block_options); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + // Initialize the Block encoder. + ret = lzma_block_encoder_init(&thr->block_encoder, + thr->allocator, &thr->block_options); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + size_t in_pos = 0; + size_t in_size = 0; + + thr->outbuf->size = thr->block_options.header_size; + const size_t out_size = thr->coder->outq.buf_size_max; + + do { + mythread_sync(thr->mutex) { + // Store in_pos and out_pos into *thr so that + // an application may read them via + // lzma_get_progress() to get progress information. + // + // NOTE: These aren't updated when the encoding + // finishes. Instead, the final values are taken + // later from thr->outbuf. + thr->progress_in = in_pos; + thr->progress_out = thr->outbuf->size; + + while (in_size == thr->in_size + && thr->state == THR_RUN) + mythread_cond_wait(&thr->cond, &thr->mutex); + + state = thr->state; + in_size = thr->in_size; + } + + // Return if we were asked to stop or exit. + if (state >= THR_STOP) + return state; + + lzma_action action = state == THR_FINISH + ? LZMA_FINISH : LZMA_RUN; + + // Limit the amount of input given to the Block encoder + // at once. This way this thread can react fairly quickly + // if the main thread wants us to stop or exit. + static const size_t in_chunk_max = 16384; + size_t in_limit = in_size; + if (in_size - in_pos > in_chunk_max) { + in_limit = in_pos + in_chunk_max; + action = LZMA_RUN; + } + + ret = thr->block_encoder.code( + thr->block_encoder.coder, thr->allocator, + thr->in, &in_pos, in_limit, thr->outbuf->buf, + &thr->outbuf->size, out_size, action); + } while (ret == LZMA_OK && thr->outbuf->size < out_size); + + switch (ret) { + case LZMA_STREAM_END: + assert(state == THR_FINISH); + + // Encode the Block Header. By doing it after + // the compression, we can store the Compressed Size + // and Uncompressed Size fields. + ret = lzma_block_header_encode(&thr->block_options, + thr->outbuf->buf); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + break; + + case LZMA_OK: + // The data was incompressible. Encode it using uncompressed + // LZMA2 chunks. + // + // First wait that we have gotten all the input. + mythread_sync(thr->mutex) { + while (thr->state == THR_RUN) + mythread_cond_wait(&thr->cond, &thr->mutex); + + state = thr->state; + in_size = thr->in_size; + } + + if (state >= THR_STOP) + return state; + + // Do the encoding. This takes care of the Block Header too. + thr->outbuf->size = 0; + ret = lzma_block_uncomp_encode(&thr->block_options, + thr->in, in_size, thr->outbuf->buf, + &thr->outbuf->size, out_size); + + // It shouldn't fail. + if (ret != LZMA_OK) { + worker_error(thr, LZMA_PROG_ERROR); + return THR_STOP; + } + + break; + + default: + worker_error(thr, ret); + return THR_STOP; + } + + // Set the size information that will be read by the main thread + // to write the Index field. + thr->outbuf->unpadded_size + = lzma_block_unpadded_size(&thr->block_options); + assert(thr->outbuf->unpadded_size != 0); + thr->outbuf->uncompressed_size = thr->block_options.uncompressed_size; + + return THR_FINISH; +} + + +static MYTHREAD_RET_TYPE +worker_start(void *thr_ptr) +{ + worker_thread *thr = thr_ptr; + worker_state state = THR_IDLE; // Init to silence a warning + + while (true) { + // Wait for work. + mythread_sync(thr->mutex) { + while (true) { + // The thread is already idle so if we are + // requested to stop, just set the state. + if (thr->state == THR_STOP) { + thr->state = THR_IDLE; + mythread_cond_signal(&thr->cond); + } + + state = thr->state; + if (state != THR_IDLE) + break; + + mythread_cond_wait(&thr->cond, &thr->mutex); + } + } + + assert(state != THR_IDLE); + assert(state != THR_STOP); + + if (state <= THR_FINISH) + state = worker_encode(thr, state); + + if (state == THR_EXIT) + break; + + // Mark the thread as idle unless the main thread has + // told us to exit. Signal is needed for the case + // where the main thread is waiting for the threads to stop. + mythread_sync(thr->mutex) { + if (thr->state != THR_EXIT) { + thr->state = THR_IDLE; + mythread_cond_signal(&thr->cond); + } + } + + mythread_sync(thr->coder->mutex) { + // Mark the output buffer as finished if + // no errors occurred. + thr->outbuf->finished = state == THR_FINISH; + + // Update the main progress info. + thr->coder->progress_in + += thr->outbuf->uncompressed_size; + thr->coder->progress_out += thr->outbuf->size; + thr->progress_in = 0; + thr->progress_out = 0; + + // Return this thread to the stack of free threads. + thr->next = thr->coder->threads_free; + thr->coder->threads_free = thr; + + mythread_cond_signal(&thr->coder->cond); + } + } + + // Exiting, free the resources. + mythread_mutex_destroy(&thr->mutex); + mythread_cond_destroy(&thr->cond); + + lzma_next_end(&thr->block_encoder, thr->allocator); + lzma_free(thr->in, thr->allocator); + return MYTHREAD_RET_VALUE; +} + + +/// Make the threads stop but not exit. Optionally wait for them to stop. +static void +threads_stop(lzma_stream_coder *coder, bool wait_for_threads) +{ + // Tell the threads to stop. + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + coder->threads[i].state = THR_STOP; + mythread_cond_signal(&coder->threads[i].cond); + } + } + + if (!wait_for_threads) + return; + + // Wait for the threads to settle in the idle state. + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + while (coder->threads[i].state != THR_IDLE) + mythread_cond_wait(&coder->threads[i].cond, + &coder->threads[i].mutex); + } + } + + return; +} + + +/// Stop the threads and free the resources associated with them. +/// Wait until the threads have exited. +static void +threads_end(lzma_stream_coder *coder, const lzma_allocator *allocator) +{ + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + coder->threads[i].state = THR_EXIT; + mythread_cond_signal(&coder->threads[i].cond); + } + } + + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + int ret = mythread_join(coder->threads[i].thread_id); + assert(ret == 0); + (void)ret; + } + + lzma_free(coder->threads, allocator); + return; +} + + +/// Initialize a new worker_thread structure and create a new thread. +static lzma_ret +initialize_new_thread(lzma_stream_coder *coder, + const lzma_allocator *allocator) +{ + worker_thread *thr = &coder->threads[coder->threads_initialized]; + + thr->in = lzma_alloc(coder->block_size, allocator); + if (thr->in == NULL) + return LZMA_MEM_ERROR; + + if (mythread_mutex_init(&thr->mutex)) + goto error_mutex; + + if (mythread_cond_init(&thr->cond)) + goto error_cond; + + thr->state = THR_IDLE; + thr->allocator = allocator; + thr->coder = coder; + thr->progress_in = 0; + thr->progress_out = 0; + thr->block_encoder = LZMA_NEXT_CODER_INIT; + + if (mythread_create(&thr->thread_id, &worker_start, thr)) + goto error_thread; + + ++coder->threads_initialized; + coder->thr = thr; + + return LZMA_OK; + +error_thread: + mythread_cond_destroy(&thr->cond); + +error_cond: + mythread_mutex_destroy(&thr->mutex); + +error_mutex: + lzma_free(thr->in, allocator); + return LZMA_MEM_ERROR; +} + + +static lzma_ret +get_thread(lzma_stream_coder *coder, const lzma_allocator *allocator) +{ + // If there are no free output subqueues, there is no + // point to try getting a thread. + if (!lzma_outq_has_buf(&coder->outq)) + return LZMA_OK; + + // If there is a free structure on the stack, use it. + mythread_sync(coder->mutex) { + if (coder->threads_free != NULL) { + coder->thr = coder->threads_free; + coder->threads_free = coder->threads_free->next; + } + } + + if (coder->thr == NULL) { + // If there are no uninitialized structures left, return. + if (coder->threads_initialized == coder->threads_max) + return LZMA_OK; + + // Initialize a new thread. + return_if_error(initialize_new_thread(coder, allocator)); + } + + // Reset the parts of the thread state that have to be done + // in the main thread. + mythread_sync(coder->thr->mutex) { + coder->thr->state = THR_RUN; + coder->thr->in_size = 0; + coder->thr->outbuf = lzma_outq_get_buf(&coder->outq); + mythread_cond_signal(&coder->thr->cond); + } + + return LZMA_OK; +} + + +static lzma_ret +stream_encode_in(lzma_stream_coder *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, lzma_action action) +{ + while (*in_pos < in_size + || (coder->thr != NULL && action != LZMA_RUN)) { + if (coder->thr == NULL) { + // Get a new thread. + const lzma_ret ret = get_thread(coder, allocator); + if (coder->thr == NULL) + return ret; + } + + // Copy the input data to thread's buffer. + size_t thr_in_size = coder->thr->in_size; + lzma_bufcpy(in, in_pos, in_size, coder->thr->in, + &thr_in_size, coder->block_size); + + // Tell the Block encoder to finish if + // - it has got block_size bytes of input; or + // - all input was used and LZMA_FINISH, LZMA_FULL_FLUSH, + // or LZMA_FULL_BARRIER was used. + // + // TODO: LZMA_SYNC_FLUSH and LZMA_SYNC_BARRIER. + const bool finish = thr_in_size == coder->block_size + || (*in_pos == in_size && action != LZMA_RUN); + + bool block_error = false; + + mythread_sync(coder->thr->mutex) { + if (coder->thr->state == THR_IDLE) { + // Something has gone wrong with the Block + // encoder. It has set coder->thread_error + // which we will read a few lines later. + block_error = true; + } else { + // Tell the Block encoder its new amount + // of input and update the state if needed. + coder->thr->in_size = thr_in_size; + + if (finish) + coder->thr->state = THR_FINISH; + + mythread_cond_signal(&coder->thr->cond); + } + } + + if (block_error) { + lzma_ret ret; + + mythread_sync(coder->mutex) { + ret = coder->thread_error; + } + + return ret; + } + + if (finish) + coder->thr = NULL; + } + + return LZMA_OK; +} + + +/// Wait until more input can be consumed, more output can be read, or +/// an optional timeout is reached. +static bool +wait_for_work(lzma_stream_coder *coder, mythread_condtime *wait_abs, + bool *has_blocked, bool has_input) +{ + if (coder->timeout != 0 && !*has_blocked) { + // Every time when stream_encode_mt() is called via + // lzma_code(), *has_blocked starts as false. We set it + // to true here and calculate the absolute time when + // we must return if there's nothing to do. + // + // The idea of *has_blocked is to avoid unneeded calls + // to mythread_condtime_set(), which may do a syscall + // depending on the operating system. + *has_blocked = true; + mythread_condtime_set(wait_abs, &coder->cond, coder->timeout); + } + + bool timed_out = false; + + mythread_sync(coder->mutex) { + // There are four things that we wait. If one of them + // becomes possible, we return. + // - If there is input left, we need to get a free + // worker thread and an output buffer for it. + // - Data ready to be read from the output queue. + // - A worker thread indicates an error. + // - Time out occurs. + while ((!has_input || coder->threads_free == NULL + || !lzma_outq_has_buf(&coder->outq)) + && !lzma_outq_is_readable(&coder->outq) + && coder->thread_error == LZMA_OK + && !timed_out) { + if (coder->timeout != 0) + timed_out = mythread_cond_timedwait( + &coder->cond, &coder->mutex, + wait_abs) != 0; + else + mythread_cond_wait(&coder->cond, + &coder->mutex); + } + } + + return timed_out; +} + + +static lzma_ret +stream_encode_mt(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + lzma_stream_coder *coder = coder_ptr; + + switch (coder->sequence) { + case SEQ_STREAM_HEADER: + lzma_bufcpy(coder->header, &coder->header_pos, + sizeof(coder->header), + out, out_pos, out_size); + if (coder->header_pos < sizeof(coder->header)) + return LZMA_OK; + + coder->header_pos = 0; + coder->sequence = SEQ_BLOCK; + + // Fall through + + case SEQ_BLOCK: { + // Initialized to silence warnings. + lzma_vli unpadded_size = 0; + lzma_vli uncompressed_size = 0; + lzma_ret ret = LZMA_OK; + + // These are for wait_for_work(). + bool has_blocked = false; + mythread_condtime wait_abs; + + while (true) { + mythread_sync(coder->mutex) { + // Check for Block encoder errors. + ret = coder->thread_error; + if (ret != LZMA_OK) { + assert(ret != LZMA_STREAM_END); + break; + } + + // Try to read compressed data to out[]. + ret = lzma_outq_read(&coder->outq, + out, out_pos, out_size, + &unpadded_size, + &uncompressed_size); + } + + if (ret == LZMA_STREAM_END) { + // End of Block. Add it to the Index. + ret = lzma_index_append(coder->index, + allocator, unpadded_size, + uncompressed_size); + + // If we didn't fill the output buffer yet, + // try to read more data. Maybe the next + // outbuf has been finished already too. + if (*out_pos < out_size) + continue; + } + + if (ret != LZMA_OK) { + // coder->thread_error was set or + // lzma_index_append() failed. + threads_stop(coder, false); + return ret; + } + + // Try to give uncompressed data to a worker thread. + ret = stream_encode_in(coder, allocator, + in, in_pos, in_size, action); + if (ret != LZMA_OK) { + threads_stop(coder, false); + return ret; + } + + // See if we should wait or return. + // + // TODO: LZMA_SYNC_FLUSH and LZMA_SYNC_BARRIER. + if (*in_pos == in_size) { + // LZMA_RUN: More data is probably coming + // so return to let the caller fill the + // input buffer. + if (action == LZMA_RUN) + return LZMA_OK; + + // LZMA_FULL_BARRIER: The same as with + // LZMA_RUN but tell the caller that the + // barrier was completed. + if (action == LZMA_FULL_BARRIER) + return LZMA_STREAM_END; + + // Finishing or flushing isn't completed until + // all input data has been encoded and copied + // to the output buffer. + if (lzma_outq_is_empty(&coder->outq)) { + // LZMA_FINISH: Continue to encode + // the Index field. + if (action == LZMA_FINISH) + break; + + // LZMA_FULL_FLUSH: Return to tell + // the caller that flushing was + // completed. + if (action == LZMA_FULL_FLUSH) + return LZMA_STREAM_END; + } + } + + // Return if there is no output space left. + // This check must be done after testing the input + // buffer, because we might want to use a different + // return code. + if (*out_pos == out_size) + return LZMA_OK; + + // Neither in nor out has been used completely. + // Wait until there's something we can do. + if (wait_for_work(coder, &wait_abs, &has_blocked, + *in_pos < in_size)) + return LZMA_TIMED_OUT; + } + + // All Blocks have been encoded and the threads have stopped. + // Prepare to encode the Index field. + return_if_error(lzma_index_encoder_init( + &coder->index_encoder, allocator, + coder->index)); + coder->sequence = SEQ_INDEX; + + // Update the progress info to take the Index and + // Stream Footer into account. Those are very fast to encode + // so in terms of progress information they can be thought + // to be ready to be copied out. + coder->progress_out += lzma_index_size(coder->index) + + LZMA_STREAM_HEADER_SIZE; + } + + // Fall through + + case SEQ_INDEX: { + // Call the Index encoder. It doesn't take any input, so + // those pointers can be NULL. + const lzma_ret ret = coder->index_encoder.code( + coder->index_encoder.coder, allocator, + NULL, NULL, 0, + out, out_pos, out_size, LZMA_RUN); + if (ret != LZMA_STREAM_END) + return ret; + + // Encode the Stream Footer into coder->buffer. + coder->stream_flags.backward_size + = lzma_index_size(coder->index); + if (lzma_stream_footer_encode(&coder->stream_flags, + coder->header) != LZMA_OK) + return LZMA_PROG_ERROR; + + coder->sequence = SEQ_STREAM_FOOTER; + } + + // Fall through + + case SEQ_STREAM_FOOTER: + lzma_bufcpy(coder->header, &coder->header_pos, + sizeof(coder->header), + out, out_pos, out_size); + return coder->header_pos < sizeof(coder->header) + ? LZMA_OK : LZMA_STREAM_END; + } + + assert(0); + return LZMA_PROG_ERROR; +} + + +static void +stream_encoder_mt_end(void *coder_ptr, const lzma_allocator *allocator) +{ + lzma_stream_coder *coder = coder_ptr; + + // Threads must be killed before the output queue can be freed. + threads_end(coder, allocator); + lzma_outq_end(&coder->outq, allocator); + + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + lzma_free(coder->filters[i].options, allocator); + + lzma_next_end(&coder->index_encoder, allocator); + lzma_index_end(coder->index, allocator); + + mythread_cond_destroy(&coder->cond); + mythread_mutex_destroy(&coder->mutex); + + lzma_free(coder, allocator); + return; +} + + +/// Options handling for lzma_stream_encoder_mt_init() and +/// lzma_stream_encoder_mt_memusage() +static lzma_ret +get_options(const lzma_mt *options, lzma_options_easy *opt_easy, + const lzma_filter **filters, uint64_t *block_size, + uint64_t *outbuf_size_max) +{ + // Validate some of the options. + if (options == NULL) + return LZMA_PROG_ERROR; + + if (options->flags != 0 || options->threads == 0 + || options->threads > LZMA_THREADS_MAX) + return LZMA_OPTIONS_ERROR; + + if (options->filters != NULL) { + // Filter chain was given, use it as is. + *filters = options->filters; + } else { + // Use a preset. + if (lzma_easy_preset(opt_easy, options->preset)) + return LZMA_OPTIONS_ERROR; + + *filters = opt_easy->filters; + } + + // Block size + if (options->block_size > 0) { + if (options->block_size > BLOCK_SIZE_MAX) + return LZMA_OPTIONS_ERROR; + + *block_size = options->block_size; + } else { + // Determine the Block size from the filter chain. + *block_size = lzma_mt_block_size(*filters); + if (*block_size == 0) + return LZMA_OPTIONS_ERROR; + + assert(*block_size <= BLOCK_SIZE_MAX); + } + + // Calculate the maximum amount output that a single output buffer + // may need to hold. This is the same as the maximum total size of + // a Block. + *outbuf_size_max = lzma_block_buffer_bound64(*block_size); + if (*outbuf_size_max == 0) + return LZMA_MEM_ERROR; + + return LZMA_OK; +} + + +static void +get_progress(void *coder_ptr, uint64_t *progress_in, uint64_t *progress_out) +{ + lzma_stream_coder *coder = coder_ptr; + + // Lock coder->mutex to prevent finishing threads from moving their + // progress info from the worker_thread structure to lzma_stream_coder. + mythread_sync(coder->mutex) { + *progress_in = coder->progress_in; + *progress_out = coder->progress_out; + + for (size_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + *progress_in += coder->threads[i].progress_in; + *progress_out += coder->threads[i] + .progress_out; + } + } + } + + return; +} + + +static lzma_ret +stream_encoder_mt_init(lzma_next_coder *next, const lzma_allocator *allocator, + const lzma_mt *options) +{ + lzma_next_coder_init(&stream_encoder_mt_init, next, allocator); + + // Get the filter chain. + lzma_options_easy easy; + const lzma_filter *filters; + uint64_t block_size; + uint64_t outbuf_size_max; + return_if_error(get_options(options, &easy, &filters, + &block_size, &outbuf_size_max)); + +#if SIZE_MAX < UINT64_MAX + if (block_size > SIZE_MAX) + return LZMA_MEM_ERROR; +#endif + + // Validate the filter chain so that we can give an error in this + // function instead of delaying it to the first call to lzma_code(). + // The memory usage calculation verifies the filter chain as + // a side effect so we take advatange of that. + if (lzma_raw_encoder_memusage(filters) == UINT64_MAX) + return LZMA_OPTIONS_ERROR; + + // Validate the Check ID. + if ((unsigned int)(options->check) > LZMA_CHECK_ID_MAX) + return LZMA_PROG_ERROR; + + if (!lzma_check_is_supported(options->check)) + return LZMA_UNSUPPORTED_CHECK; + + // Allocate and initialize the base structure if needed. + lzma_stream_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) + return LZMA_MEM_ERROR; + + next->coder = coder; + + // For the mutex and condition variable initializations + // the error handling has to be done here because + // stream_encoder_mt_end() doesn't know if they have + // already been initialized or not. + if (mythread_mutex_init(&coder->mutex)) { + lzma_free(coder, allocator); + next->coder = NULL; + return LZMA_MEM_ERROR; + } + + if (mythread_cond_init(&coder->cond)) { + mythread_mutex_destroy(&coder->mutex); + lzma_free(coder, allocator); + next->coder = NULL; + return LZMA_MEM_ERROR; + } + + next->code = &stream_encode_mt; + next->end = &stream_encoder_mt_end; + next->get_progress = &get_progress; +// next->update = &stream_encoder_mt_update; + + coder->filters[0].id = LZMA_VLI_UNKNOWN; + coder->index_encoder = LZMA_NEXT_CODER_INIT; + coder->index = NULL; + memzero(&coder->outq, sizeof(coder->outq)); + coder->threads = NULL; + coder->threads_max = 0; + coder->threads_initialized = 0; + } + + // Basic initializations + coder->sequence = SEQ_STREAM_HEADER; + coder->block_size = (size_t)(block_size); + coder->thread_error = LZMA_OK; + coder->thr = NULL; + + // Allocate the thread-specific base structures. + assert(options->threads > 0); + if (coder->threads_max != options->threads) { + threads_end(coder, allocator); + + coder->threads = NULL; + coder->threads_max = 0; + + coder->threads_initialized = 0; + coder->threads_free = NULL; + + coder->threads = lzma_alloc( + options->threads * sizeof(worker_thread), + allocator); + if (coder->threads == NULL) + return LZMA_MEM_ERROR; + + coder->threads_max = options->threads; + } else { + // Reuse the old structures and threads. Tell the running + // threads to stop and wait until they have stopped. + threads_stop(coder, true); + } + + // Output queue + return_if_error(lzma_outq_init(&coder->outq, allocator, + outbuf_size_max, options->threads)); + + // Timeout + coder->timeout = options->timeout; + + // Free the old filter chain and copy the new one. + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + lzma_free(coder->filters[i].options, allocator); + + return_if_error(lzma_filters_copy( + filters, coder->filters, allocator)); + + // Index + lzma_index_end(coder->index, allocator); + coder->index = lzma_index_init(allocator); + if (coder->index == NULL) + return LZMA_MEM_ERROR; + + // Stream Header + coder->stream_flags.version = 0; + coder->stream_flags.check = options->check; + return_if_error(lzma_stream_header_encode( + &coder->stream_flags, coder->header)); + + coder->header_pos = 0; + + // Progress info + coder->progress_in = 0; + coder->progress_out = LZMA_STREAM_HEADER_SIZE; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_stream_encoder_mt(lzma_stream *strm, const lzma_mt *options) +{ + lzma_next_strm_init(stream_encoder_mt_init, strm, options); + + strm->internal->supported_actions[LZMA_RUN] = true; +// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_BARRIER] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} + + +// This function name is a monster but it's consistent with the older +// monster names. :-( 31 chars is the max that C99 requires so in that +// sense it's not too long. ;-) +extern LZMA_API(uint64_t) +lzma_stream_encoder_mt_memusage(const lzma_mt *options) +{ + lzma_options_easy easy; + const lzma_filter *filters; + uint64_t block_size; + uint64_t outbuf_size_max; + + if (get_options(options, &easy, &filters, &block_size, + &outbuf_size_max) != LZMA_OK) + return UINT64_MAX; + + // Memory usage of the input buffers + const uint64_t inbuf_memusage = options->threads * block_size; + + // Memory usage of the filter encoders + uint64_t filters_memusage = lzma_raw_encoder_memusage(filters); + if (filters_memusage == UINT64_MAX) + return UINT64_MAX; + + filters_memusage *= options->threads; + + // Memory usage of the output queue + const uint64_t outq_memusage = lzma_outq_memusage( + outbuf_size_max, options->threads); + if (outq_memusage == UINT64_MAX) + return UINT64_MAX; + + // Sum them with overflow checking. + uint64_t total_memusage = LZMA_MEMUSAGE_BASE + + sizeof(lzma_stream_coder) + + options->threads * sizeof(worker_thread); + + if (UINT64_MAX - total_memusage < inbuf_memusage) + return UINT64_MAX; + + total_memusage += inbuf_memusage; + + if (UINT64_MAX - total_memusage < filters_memusage) + return UINT64_MAX; + + total_memusage += filters_memusage; + + if (UINT64_MAX - total_memusage < outq_memusage) + return UINT64_MAX; + + return total_memusage + outq_memusage; +} diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c index 8cf48a4..1bc2f97 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c @@ -30,15 +30,13 @@ stream_flags_decode(lzma_stream_flags *options, const uint8_t *in) extern LZMA_API(lzma_ret) lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in) { - uint32_t crc; - // Magic if (memcmp(in, lzma_header_magic, sizeof(lzma_header_magic)) != 0) return LZMA_FORMAT_ERROR; // Verify the CRC32 so we can distinguish between corrupt // and unsupported files. - crc = lzma_crc32(in + sizeof(lzma_header_magic), + const uint32_t crc = lzma_crc32(in + sizeof(lzma_header_magic), LZMA_STREAM_FLAGS_SIZE, 0); if (crc != unaligned_read32le(in + sizeof(lzma_header_magic) + LZMA_STREAM_FLAGS_SIZE)) @@ -61,15 +59,13 @@ lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in) extern LZMA_API(lzma_ret) lzma_stream_footer_decode(lzma_stream_flags *options, const uint8_t *in) { - uint32_t crc; - // Magic if (memcmp(in + sizeof(uint32_t) * 2 + LZMA_STREAM_FLAGS_SIZE, lzma_footer_magic, sizeof(lzma_footer_magic)) != 0) return LZMA_FORMAT_ERROR; // CRC32 - crc = lzma_crc32(in + sizeof(uint32_t), + const uint32_t crc = lzma_crc32(in + sizeof(uint32_t), sizeof(uint32_t) + LZMA_STREAM_FLAGS_SIZE, 0); if (crc != unaligned_read32le(in)) return LZMA_DATA_ERROR; diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c index 290339e..4e71715 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c @@ -29,8 +29,6 @@ stream_flags_encode(const lzma_stream_flags *options, uint8_t *out) extern LZMA_API(lzma_ret) lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) { - uint32_t crc; - assert(sizeof(lzma_header_magic) + LZMA_STREAM_FLAGS_SIZE + 4 == LZMA_STREAM_HEADER_SIZE); @@ -45,7 +43,7 @@ lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) return LZMA_PROG_ERROR; // CRC32 of the Stream Header - crc = lzma_crc32(out + sizeof(lzma_header_magic), + const uint32_t crc = lzma_crc32(out + sizeof(lzma_header_magic), LZMA_STREAM_FLAGS_SIZE, 0); unaligned_write32le(out + sizeof(lzma_header_magic) @@ -58,8 +56,6 @@ lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) extern LZMA_API(lzma_ret) lzma_stream_footer_encode(const lzma_stream_flags *options, uint8_t *out) { - uint32_t crc; - assert(2 * 4 + LZMA_STREAM_FLAGS_SIZE + sizeof(lzma_footer_magic) == LZMA_STREAM_HEADER_SIZE); @@ -77,7 +73,7 @@ lzma_stream_footer_encode(const lzma_stream_flags *options, uint8_t *out) return LZMA_PROG_ERROR; // CRC32 - crc = lzma_crc32( + const uint32_t crc = lzma_crc32( out + 4, 4 + LZMA_STREAM_FLAGS_SIZE, 0); unaligned_write32le(out, crc); diff --git a/Utilities/cmliblzma/liblzma/common/vli_decoder.c b/Utilities/cmliblzma/liblzma/common/vli_decoder.c index 1c66384..c181828 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/vli_decoder.c @@ -14,8 +14,8 @@ extern LZMA_API(lzma_ret) -lzma_vli_decode(lzma_vli *LZMA_RESTRICT vli, size_t *vli_pos, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lzma_vli_decode(lzma_vli *restrict vli, size_t *vli_pos, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size) { // If we haven't been given vli_pos, work in single-call mode. diff --git a/Utilities/cmliblzma/liblzma/common/vli_encoder.c b/Utilities/cmliblzma/liblzma/common/vli_encoder.c index 09e90cb..f864269 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/vli_encoder.c @@ -15,7 +15,7 @@ extern LZMA_API(lzma_ret) lzma_vli_encode(lzma_vli vli, size_t *vli_pos, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { // If we haven't been given vli_pos, work in single-call mode. diff --git a/Utilities/cmliblzma/liblzma/common/vli_size.c b/Utilities/cmliblzma/liblzma/common/vli_size.c index 8b931e4..ec1b4fa 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_size.c +++ b/Utilities/cmliblzma/liblzma/common/vli_size.c @@ -16,11 +16,10 @@ extern LZMA_API(uint32_t) lzma_vli_size(lzma_vli vli) { - uint32_t i = 0; - if (vli > LZMA_VLI_MAX) return 0; + uint32_t i = 0; do { vli >>= 7; ++i; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_common.c b/Utilities/cmliblzma/liblzma/delta/delta_common.c index 803e674..4768201 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_common.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_common.c @@ -15,8 +15,9 @@ static void -delta_coder_end(lzma_coder *coder, lzma_allocator *allocator) +delta_coder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_delta_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -24,20 +25,21 @@ delta_coder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_delta_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { - const lzma_options_delta *opt; - // Allocate memory for the decoder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_delta_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_delta_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; + // End function is the same for encoder and decoder. next->end = &delta_coder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Validate the options. @@ -45,16 +47,15 @@ lzma_delta_coder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_OPTIONS_ERROR; // Set the delta distance. - opt = filters[0].options; - next->coder->distance = opt->dist; + const lzma_options_delta *opt = filters[0].options; + coder->distance = opt->dist; // Initialize the rest of the variables. - next->coder->pos = 0; - memzero(next->coder->history, LZMA_DELTA_DIST_MAX); + coder->pos = 0; + memzero(coder->history, LZMA_DELTA_DIST_MAX); // Initialize the next decoder in the chain, if any. - return lzma_next_filter_init(&next->coder->next, - allocator, filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } @@ -68,5 +69,5 @@ lzma_delta_coder_memusage(const void *options) || opt->dist > LZMA_DELTA_DIST_MAX) return UINT64_MAX; - return sizeof(lzma_coder); + return sizeof(lzma_delta_coder); } diff --git a/Utilities/cmliblzma/liblzma/delta/delta_decoder.c b/Utilities/cmliblzma/liblzma/delta/delta_decoder.c index 28df727..6859afa 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_decoder.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_decoder.c @@ -15,12 +15,11 @@ static void -decode_buffer(lzma_coder *coder, uint8_t *buffer, size_t size) +decode_buffer(lzma_delta_coder *coder, uint8_t *buffer, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { buffer[i] += coder->history[(distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = buffer[i]; } @@ -28,17 +27,18 @@ decode_buffer(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -delta_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +delta_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { - const size_t out_start = *out_pos; - lzma_ret ret; + lzma_delta_coder *coder = coder_ptr; assert(coder->next.code != NULL); - ret = coder->next.code(coder->next.coder, allocator, + const size_t out_start = *out_pos; + + const lzma_ret ret = coder->next.code(coder->next.coder, allocator, in, in_pos, in_size, out, out_pos, out_size, action); @@ -49,7 +49,7 @@ delta_decode(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { next->code = &delta_decode; @@ -58,15 +58,14 @@ lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_delta_props_decode(void **options, lzma_allocator *allocator, +lzma_delta_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_delta *opt; - if (props_size != 1) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_delta), allocator); + lzma_options_delta *opt + = lzma_alloc(sizeof(lzma_options_delta), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_decoder.h b/Utilities/cmliblzma/liblzma/delta/delta_decoder.h index ae89acc..ad89cc6 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_decoder.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_decoder.h @@ -16,10 +16,11 @@ #include "delta_common.h" extern lzma_ret lzma_delta_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_delta_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/delta/delta_encoder.c b/Utilities/cmliblzma/liblzma/delta/delta_encoder.c index a39c154..3841651 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_encoder.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_encoder.c @@ -18,13 +18,12 @@ /// is the first filter in the chain (and thus the last filter in the /// encoder's filter stack). static void -copy_and_encode(lzma_coder *coder, - const uint8_t *LZMA_RESTRICT in, uint8_t *LZMA_RESTRICT out, size_t size) +copy_and_encode(lzma_delta_coder *coder, + const uint8_t *restrict in, uint8_t *restrict out, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { const uint8_t tmp = coder->history[ (distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = in[i]; @@ -36,12 +35,11 @@ copy_and_encode(lzma_coder *coder, /// Encodes the data in place. This is used when we are the last filter /// in the chain (and thus non-last filter in the encoder's filter stack). static void -encode_in_place(lzma_coder *coder, uint8_t *buffer, size_t size) +encode_in_place(lzma_delta_coder *coder, uint8_t *buffer, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { const uint8_t tmp = coder->history[ (distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = buffer[i]; @@ -51,11 +49,13 @@ encode_in_place(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -delta_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +delta_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_delta_coder *coder = coder_ptr; + lzma_ret ret; if (coder->next.code == NULL) { @@ -86,10 +86,12 @@ delta_encode(lzma_coder *coder, lzma_allocator *allocator, static lzma_ret -delta_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +delta_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_delta_coder *coder = coder_ptr; + // Delta doesn't and will never support changing the options in // the middle of encoding. If the app tries to change them, we // simply ignore them. @@ -99,7 +101,7 @@ delta_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { next->code = &delta_encode; @@ -111,13 +113,12 @@ lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out) { - const lzma_options_delta *opt = options; - // The caller must have already validated the options, so it's // LZMA_PROG_ERROR if they are invalid. if (lzma_delta_coder_memusage(options) == UINT64_MAX) return LZMA_PROG_ERROR; + const lzma_options_delta *opt = options; out[0] = opt->dist - LZMA_DELTA_DIST_MIN; return LZMA_OK; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_encoder.h b/Utilities/cmliblzma/liblzma/delta/delta_encoder.h index a447862..4ab9847 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_encoder.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_encoder.h @@ -16,7 +16,8 @@ #include "delta_common.h" extern lzma_ret lzma_delta_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out); diff --git a/Utilities/cmliblzma/liblzma/delta/delta_private.h b/Utilities/cmliblzma/liblzma/delta/delta_private.h index 62b7fed..0d6cb38 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_private.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_private.h @@ -15,7 +15,7 @@ #include "delta_common.h" -struct lzma_coder_s { +typedef struct { /// Next coder in the chain lzma_next_coder next; @@ -27,11 +27,11 @@ struct lzma_coder_s { /// Buffer to hold history of the original data uint8_t history[LZMA_DELTA_DIST_MAX]; -}; +} lzma_delta_coder; extern lzma_ret lzma_delta_coder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/liblzma.pc.in b/Utilities/cmliblzma/liblzma/liblzma.pc.in index 7f11f1a..9fa4891 100644 --- a/Utilities/cmliblzma/liblzma/liblzma.pc.in +++ b/Utilities/cmliblzma/liblzma/liblzma.pc.in @@ -16,4 +16,4 @@ URL: @PACKAGE_URL@ Version: @PACKAGE_VERSION@ Cflags: -I${includedir} Libs: -L${libdir} -llzma -Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ +Libs.private: @PTHREAD_CFLAGS@ @LIBS@ diff --git a/Utilities/cmliblzma/liblzma/lz/lz_decoder.c b/Utilities/cmliblzma/liblzma/lz/lz_decoder.c index 9fa1bdc..c708644 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_decoder.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_decoder.c @@ -20,7 +20,7 @@ #include "lz_decoder.h" -struct lzma_coder_s { +typedef struct { /// Dictionary (history buffer) lzma_dict dict; @@ -48,7 +48,7 @@ struct lzma_coder_s { size_t size; uint8_t buffer[LZMA_BUFFER_SIZE]; } temp; -}; +} lzma_coder; static void @@ -64,22 +64,18 @@ lz_decoder_reset(lzma_coder *coder) static lzma_ret decode_buffer(lzma_coder *coder, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size) + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) { while (true) { - size_t copy_size; - size_t dict_start; - lzma_ret ret; - // Wrap the dictionary if needed. if (coder->dict.pos == coder->dict.size) coder->dict.pos = 0; // Store the current dictionary position. It is needed to know // where to start copying to the out[] buffer. - dict_start = coder->dict.pos; + const size_t dict_start = coder->dict.pos; // Calculate how much we allow coder->lz.code() to decode. // It must not decode past the end of the dictionary @@ -90,13 +86,13 @@ decode_buffer(lzma_coder *coder, coder->dict.size - coder->dict.pos); // Call the coder->lz.code() to do the actual decoding. - ret = coder->lz.code( + const lzma_ret ret = coder->lz.code( coder->lz.coder, &coder->dict, in, in_pos, in_size); // Copy the decoded data from the dictionary to the out[] // buffer. - copy_size = coder->dict.pos - dict_start; + const size_t copy_size = coder->dict.pos - dict_start; assert(copy_size <= out_size - *out_pos); memcpy(out + *out_pos, coder->dict.buf + dict_start, copy_size); @@ -129,13 +125,15 @@ decode_buffer(lzma_coder *coder, static lzma_ret -lz_decode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +lz_decode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_coder *coder = coder_ptr; + if (coder->next.code == NULL) return decode_buffer(coder, in, in_pos, in_size, out, out_pos, out_size); @@ -143,15 +141,13 @@ lz_decode(lzma_coder *coder, // We aren't the last coder in the chain, we need to decode // our input to a temporary buffer. while (*out_pos < out_size) { - lzma_ret ret; - // Fill the temporary buffer if it is empty. if (!coder->next_finished && coder->temp.pos == coder->temp.size) { coder->temp.pos = 0; coder->temp.size = 0; - ret = coder->next.code( + const lzma_ret ret = coder->next.code( coder->next.coder, allocator, in, in_pos, in_size, coder->temp.buffer, &coder->temp.size, @@ -173,7 +169,7 @@ lz_decode(lzma_coder *coder, return LZMA_OK; } - ret = decode_buffer(coder, coder->temp.buffer, + const lzma_ret ret = decode_buffer(coder, coder->temp.buffer, &coder->temp.pos, coder->temp.size, out, out_pos, out_size); @@ -190,8 +186,10 @@ lz_decode(lzma_coder *coder, static void -lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +lz_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_coder *coder = coder_ptr; + lzma_next_end(&coder->next, allocator); lzma_free(coder->dict.buf, allocator); @@ -206,32 +204,33 @@ lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_decoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)) { - lzma_lz_options lz_options; - // Allocate the base structure if it isn't already allocated. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &lz_decode; next->end = &lz_decoder_end; - next->coder->dict.buf = NULL; - next->coder->dict.size = 0; - next->coder->lz = LZMA_LZ_DECODER_INIT; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->dict.buf = NULL; + coder->dict.size = 0; + coder->lz = LZMA_LZ_DECODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Allocate and initialize the LZ-based decoder. It will also give // us the dictionary size. - return_if_error(lz_init(&next->coder->lz, allocator, + lzma_lz_options lz_options; + return_if_error(lz_init(&coder->lz, allocator, filters[0].options, &lz_options)); // If the dictionary size is very small, increase it to 4096 bytes. @@ -255,14 +254,14 @@ lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, lz_options.dict_size = (lz_options.dict_size + 15) & ~((size_t)(15)); // Allocate and initialize the dictionary. - if (next->coder->dict.size != lz_options.dict_size) { - lzma_free(next->coder->dict.buf, allocator); - next->coder->dict.buf + if (coder->dict.size != lz_options.dict_size) { + lzma_free(coder->dict.buf, allocator); + coder->dict.buf = lzma_alloc(lz_options.dict_size, allocator); - if (next->coder->dict.buf == NULL) + if (coder->dict.buf == NULL) return LZMA_MEM_ERROR; - next->coder->dict.size = lz_options.dict_size; + coder->dict.size = lz_options.dict_size; } lz_decoder_reset(next->coder); @@ -275,21 +274,20 @@ lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, const size_t copy_size = my_min(lz_options.preset_dict_size, lz_options.dict_size); const size_t offset = lz_options.preset_dict_size - copy_size; - memcpy(next->coder->dict.buf, lz_options.preset_dict + offset, + memcpy(coder->dict.buf, lz_options.preset_dict + offset, copy_size); - next->coder->dict.pos = copy_size; - next->coder->dict.full = copy_size; + coder->dict.pos = copy_size; + coder->dict.full = copy_size; } // Miscellaneous initializations - next->coder->next_finished = false; - next->coder->this_finished = false; - next->coder->temp.pos = 0; - next->coder->temp.size = 0; + coder->next_finished = false; + coder->this_finished = false; + coder->temp.pos = 0; + coder->temp.size = 0; // Initialize the next filter in the chain, if any. - return lzma_next_filter_init(&next->coder->next, allocator, - filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } @@ -301,7 +299,8 @@ lzma_lz_decoder_memusage(size_t dictionary_size) extern void -lzma_lz_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) +lzma_lz_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) { + lzma_coder *coder = coder_ptr; coder->lz.set_uncompressed(coder->lz.coder, uncompressed_size); } diff --git a/Utilities/cmliblzma/liblzma/lz/lz_decoder.h b/Utilities/cmliblzma/liblzma/lz/lz_decoder.h index 76011f2..754ccf3 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_decoder.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_decoder.h @@ -53,45 +53,45 @@ typedef struct { typedef struct { /// Data specific to the LZ-based decoder - lzma_coder *coder; + void *coder; /// Function to decode from in[] to *dict - lzma_ret (*code)(lzma_coder *LZMA_RESTRICT coder, - lzma_dict *LZMA_RESTRICT dict, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size); + lzma_ret (*code)(void *coder, + lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size); - void (*reset)(lzma_coder *coder, const void *options); + void (*reset)(void *coder, const void *options); /// Set the uncompressed size - void (*set_uncompressed)(lzma_coder *coder, - lzma_vli uncompressed_size); + void (*set_uncompressed)(void *coder, lzma_vli uncompressed_size); /// Free allocated resources - void (*end)(lzma_coder *coder, lzma_allocator *allocator); + void (*end)(void *coder, const lzma_allocator *allocator); } lzma_lz_decoder; -static const lzma_lz_decoder LZMA_LZ_DECODER_INIT = - { - NULL, - NULL, - NULL, - NULL, - NULL, - }; +#define LZMA_LZ_DECODER_INIT \ + (lzma_lz_decoder){ \ + .coder = NULL, \ + .code = NULL, \ + .reset = NULL, \ + .set_uncompressed = NULL, \ + .end = NULL, \ + } extern lzma_ret lzma_lz_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters, + const lzma_allocator *allocator, + const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_decoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)); extern uint64_t lzma_lz_decoder_memusage(size_t dictionary_size); extern void lzma_lz_decoder_uncompressed( - lzma_coder *coder, lzma_vli uncompressed_size); + void *coder, lzma_vli uncompressed_size); ////////////////////// @@ -151,15 +151,13 @@ dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len) dict->pos += left; } else { - uint32_t copy_pos; - uint32_t copy_size; - // The bigger the dictionary, the more rare this // case occurs. We need to "wrap" the dict, thus // we might need two memcpy() to copy all the data. assert(dict->full == dict->size); - copy_pos = dict->pos - distance - 1 + dict->size; - copy_size = dict->size - copy_pos; + const uint32_t copy_pos + = dict->pos - distance - 1 + dict->size; + uint32_t copy_size = dict->size - copy_pos; if (copy_size < left) { memmove(dict->buf + dict->pos, dict->buf + copy_pos, @@ -202,9 +200,9 @@ dict_put(lzma_dict *dict, uint8_t byte) /// Copies arbitrary amount of data into the dictionary. static inline void -dict_write(lzma_dict *LZMA_RESTRICT dict, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size, - size_t *LZMA_RESTRICT left) +dict_write(lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size, + size_t *restrict left) { // NOTE: If we are being given more data than the size of the // dictionary, it could be possible to optimize the LZ decoder diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder.c b/Utilities/cmliblzma/liblzma/lz/lz_encoder.c index 1dae924..9a74b7c 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder.c @@ -20,8 +20,10 @@ # include "lz_encoder_hash_table.h" #endif +#include "memcmplen.h" -struct lzma_coder_s { + +typedef struct { /// LZ-based encoder e.g. LZMA lzma_lz_encoder lz; @@ -30,7 +32,7 @@ struct lzma_coder_s { /// Next coder in the chain lzma_next_coder next; -}; +} lzma_coder; /// \brief Moves the data in the input window to free space for new data @@ -43,18 +45,16 @@ struct lzma_coder_s { static void move_window(lzma_mf *mf) { - uint32_t move_offset; - size_t move_size; - // Align the move to a multiple of 16 bytes. Some LZ-based encoders // like LZMA use the lowest bits of mf->read_pos to know the // alignment of the uncompressed data. We also get better speed // for memmove() with aligned buffers. assert(mf->read_pos > mf->keep_size_before); - move_offset = (mf->read_pos - mf->keep_size_before) & ~UINT32_C(15); + const uint32_t move_offset + = (mf->read_pos - mf->keep_size_before) & ~UINT32_C(15); assert(mf->write_pos > move_offset); - move_size = mf->write_pos - move_offset; + const size_t move_size = mf->write_pos - move_offset; assert(move_offset + move_size <= mf->size); @@ -78,12 +78,10 @@ move_window(lzma_mf *mf) /// This function must not be called once it has returned LZMA_STREAM_END. /// static lzma_ret -fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, - size_t *in_pos, size_t in_size, lzma_action action) +fill_window(lzma_coder *coder, const lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size, + lzma_action action) { - size_t write_pos; - lzma_ret ret; - assert(coder->mf.read_pos <= coder->mf.write_pos); // Move the sliding window if needed. @@ -93,7 +91,8 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, // Maybe this is ugly, but lzma_mf uses uint32_t for most things // (which I find cleanest), but we need size_t here when filling // the history window. - write_pos = coder->mf.write_pos; + size_t write_pos = coder->mf.write_pos; + lzma_ret ret; if (coder->next.code == NULL) { // Not using a filter, simply memcpy() as much as possible. lzma_bufcpy(in, in_pos, in_size, coder->mf.buffer, @@ -111,6 +110,12 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, coder->mf.write_pos = write_pos; + // Silence Valgrind. lzma_memcmplen() can read extra bytes + // and Valgrind will give warnings if those bytes are uninitialized + // because Valgrind cannot see that the values of the uninitialized + // bytes are eventually ignored. + memzero(coder->mf.buffer + write_pos, LZMA_MEMCMPLEN_EXTRA); + // If end of stream has been reached or flushing completed, we allow // the encoder to process all the input (that is, read_pos is allowed // to reach write_pos). Otherwise we keep keep_size_after bytes @@ -134,7 +139,7 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, && coder->mf.read_pos < coder->mf.read_limit) { // Match finder may update coder->pending and expects it to // start from zero, so use a temporary variable. - const size_t pending = coder->mf.pending; + const uint32_t pending = coder->mf.pending; coder->mf.pending = 0; // Rewind read_pos so that the match finder can hash @@ -152,16 +157,16 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, static lzma_ret -lz_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lz_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_coder *coder = coder_ptr; + while (*out_pos < out_size && (*in_pos < in_size || action != LZMA_RUN)) { - lzma_ret ret; - // Read more data to coder->mf.buffer if needed. if (coder->mf.action == LZMA_RUN && coder->mf.read_pos >= coder->mf.read_limit) @@ -169,7 +174,7 @@ lz_encode(lzma_coder *coder, lzma_allocator *allocator, in, in_pos, in_size, action)); // Encode - ret = coder->lz.code(coder->lz.coder, + const lzma_ret ret = coder->lz.code(coder->lz.coder, &coder->mf, out, out_pos, out_size); if (ret != LZMA_OK) { // Setting this to LZMA_RUN for cases when we are @@ -185,17 +190,9 @@ lz_encode(lzma_coder *coder, lzma_allocator *allocator, static bool -lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, +lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator, const lzma_lz_options *lz_options) { - bool is_bt; - uint32_t new_count; - uint32_t reserve; - uint32_t old_size; - uint32_t hash_bytes; - uint32_t hs; - uint32_t old_count; - // For now, the dictionary size is limited to 1.5 GiB. This may grow // in the future if needed, but it needs a little more work than just // changing this check. @@ -221,14 +218,14 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, // to size_t. // - Memory usage calculation needs something too, e.g. use uint64_t // for mf->size. - reserve = lz_options->dict_size / 2; + uint32_t reserve = lz_options->dict_size / 2; if (reserve > (UINT32_C(1) << 30)) reserve /= 2; reserve += (lz_options->before_size + lz_options->match_len_max + lz_options->after_size) / 2 + (UINT32_C(1) << 19); - old_size = mf->size; + const uint32_t old_size = mf->size; mf->size = mf->keep_size_before + reserve + mf->keep_size_after; // Deallocate the old history buffer if it exists but has different @@ -298,11 +295,12 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, // Calculate the sizes of mf->hash and mf->son and check that // nice_len is big enough for the selected match finder. - hash_bytes = lz_options->match_finder & 0x0F; + const uint32_t hash_bytes = lz_options->match_finder & 0x0F; if (hash_bytes > mf->nice_len) return true; - is_bt = (lz_options->match_finder & 0x10) != 0; + const bool is_bt = (lz_options->match_finder & 0x10) != 0; + uint32_t hs; if (hash_bytes == 2) { hs = 0xFFFF; @@ -338,25 +336,22 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, hs += HASH_4_SIZE; */ - // If the above code calculating hs is modified, make sure that - // this assertion stays valid (UINT32_MAX / 5 is not strictly the - // exact limit). If it doesn't, you need to calculate that - // hash_size_sum + sons_count cannot overflow. - assert(hs < UINT32_MAX / 5); - - old_count = mf->hash_size_sum + mf->sons_count; - mf->hash_size_sum = hs; + const uint32_t old_hash_count = mf->hash_count; + const uint32_t old_sons_count = mf->sons_count; + mf->hash_count = hs; mf->sons_count = mf->cyclic_size; if (is_bt) mf->sons_count *= 2; - new_count = mf->hash_size_sum + mf->sons_count; - // Deallocate the old hash array if it exists and has different size // than what is needed now. - if (old_count != new_count) { + if (old_hash_count != mf->hash_count + || old_sons_count != mf->sons_count) { lzma_free(mf->hash, allocator); mf->hash = NULL; + + lzma_free(mf->son, allocator); + mf->son = NULL; } // Maximum number of match finder cycles @@ -373,16 +368,23 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, static bool -lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator, +lz_encoder_init(lzma_mf *mf, const lzma_allocator *allocator, const lzma_lz_options *lz_options) { - size_t alloc_count; - // Allocate the history buffer. if (mf->buffer == NULL) { - mf->buffer = lzma_alloc(mf->size, allocator); + // lzma_memcmplen() is used for the dictionary buffer + // so we need to allocate a few extra bytes to prevent + // it from reading past the end of the buffer. + mf->buffer = lzma_alloc(mf->size + LZMA_MEMCMPLEN_EXTRA, + allocator); if (mf->buffer == NULL) return true; + + // Keep Valgrind happy with lzma_memcmplen() and initialize + // the extra bytes whose value may get read but which will + // effectively get ignored. + memzero(mf->buffer + mf->size, LZMA_MEMCMPLEN_EXTRA); } // Use cyclic_size as initial mf->offset. This allows @@ -396,43 +398,48 @@ lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator, mf->write_pos = 0; mf->pending = 0; - // Allocate match finder's hash array. - alloc_count = mf->hash_size_sum + mf->sons_count; - #if UINT32_MAX >= SIZE_MAX / 4 // Check for integer overflow. (Huge dictionaries are not // possible on 32-bit CPU.) - if (alloc_count > SIZE_MAX / sizeof(uint32_t)) + if (mf->hash_count > SIZE_MAX / sizeof(uint32_t) + || mf->sons_count > SIZE_MAX / sizeof(uint32_t)) return true; #endif + // Allocate and initialize the hash table. Since EMPTY_HASH_VALUE + // is zero, we can use lzma_alloc_zero() or memzero() for mf->hash. + // + // We don't need to initialize mf->son, but not doing that may + // make Valgrind complain in normalization (see normalize() in + // lz_encoder_mf.c). Skipping the initialization is *very* good + // when big dictionary is used but only small amount of data gets + // actually compressed: most of the mf->son won't get actually + // allocated by the kernel, so we avoid wasting RAM and improve + // initialization speed a lot. if (mf->hash == NULL) { - mf->hash = lzma_alloc(alloc_count * sizeof(uint32_t), + mf->hash = lzma_alloc_zero(mf->hash_count * sizeof(uint32_t), + allocator); + mf->son = lzma_alloc(mf->sons_count * sizeof(uint32_t), allocator); - if (mf->hash == NULL) - return true; - } - mf->son = mf->hash + mf->hash_size_sum; - mf->cyclic_pos = 0; + if (mf->hash == NULL || mf->son == NULL) { + lzma_free(mf->hash, allocator); + mf->hash = NULL; + + lzma_free(mf->son, allocator); + mf->son = NULL; - // Initialize the hash table. Since EMPTY_HASH_VALUE is zero, we - // can use memset(). + return true; + } + } else { /* - for (uint32_t i = 0; i < hash_size_sum; ++i) - mf->hash[i] = EMPTY_HASH_VALUE; + for (uint32_t i = 0; i < mf->hash_count; ++i) + mf->hash[i] = EMPTY_HASH_VALUE; */ - memzero(mf->hash, (size_t)(mf->hash_size_sum) * sizeof(uint32_t)); + memzero(mf->hash, mf->hash_count * sizeof(uint32_t)); + } - // We don't need to initialize mf->son, but not doing that will - // make Valgrind complain in normalization (see normalize() in - // lz_encoder_mf.c). - // - // Skipping this initialization is *very* good when big dictionary is - // used but only small amount of data gets actually compressed: most - // of the mf->hash won't get actually allocated by the kernel, so - // we avoid wasting RAM and improve initialization speed a lot. - //memzero(mf->son, (size_t)(mf->sons_count) * sizeof(uint32_t)); + mf->cyclic_pos = 0; // Handle preset dictionary. if (lz_options->preset_dict != NULL @@ -457,24 +464,32 @@ extern uint64_t lzma_lz_encoder_memusage(const lzma_lz_options *lz_options) { // Old buffers must not exist when calling lz_encoder_prepare(). - lzma_mf mf = { NULL }; + lzma_mf mf = { + .buffer = NULL, + .hash = NULL, + .son = NULL, + .hash_count = 0, + .sons_count = 0, + }; // Setup the size information into mf. if (lz_encoder_prepare(&mf, NULL, lz_options)) return UINT64_MAX; // Calculate the memory usage. - return (uint64_t)(mf.hash_size_sum + mf.sons_count) - * sizeof(uint32_t) - + (uint64_t)(mf.size) + sizeof(lzma_coder); + return ((uint64_t)(mf.hash_count) + mf.sons_count) * sizeof(uint32_t) + + mf.size + sizeof(lzma_coder); } static void -lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +lz_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_coder *coder = coder_ptr; + lzma_next_end(&coder->next, allocator); + lzma_free(coder->mf.son, allocator); lzma_free(coder->mf.hash, allocator); lzma_free(coder->mf.buffer, allocator); @@ -489,10 +504,12 @@ lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lz_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +lz_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_coder *coder = coder_ptr; + if (coder->lz.options_update == NULL) return LZMA_PROG_ERROR; @@ -505,58 +522,63 @@ lz_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_lz_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lz_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_encoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)) { - lzma_lz_options lz_options; - #ifdef HAVE_SMALL // We need that the CRC32 table has been initialized. lzma_crc32_init(); #endif // Allocate and initialize the base data structure. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &lz_encode; next->end = &lz_encoder_end; next->update = &lz_encoder_update; - next->coder->lz.coder = NULL; - next->coder->lz.code = NULL; - next->coder->lz.end = NULL; - - next->coder->mf.buffer = NULL; - next->coder->mf.hash = NULL; - next->coder->mf.hash_size_sum = 0; - next->coder->mf.sons_count = 0; - - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->lz.coder = NULL; + coder->lz.code = NULL; + coder->lz.end = NULL; + + // mf.size is initialized to silence Valgrind + // when used on optimized binaries (GCC may reorder + // code in a way that Valgrind gets unhappy). + coder->mf.buffer = NULL; + coder->mf.size = 0; + coder->mf.hash = NULL; + coder->mf.son = NULL; + coder->mf.hash_count = 0; + coder->mf.sons_count = 0; + + coder->next = LZMA_NEXT_CODER_INIT; } // Initialize the LZ-based encoder. - return_if_error(lz_init(&next->coder->lz, allocator, + lzma_lz_options lz_options; + return_if_error(lz_init(&coder->lz, allocator, filters[0].options, &lz_options)); - // Setup the size information into next->coder->mf and deallocate + // Setup the size information into coder->mf and deallocate // old buffers if they have wrong size. - if (lz_encoder_prepare(&next->coder->mf, allocator, &lz_options)) + if (lz_encoder_prepare(&coder->mf, allocator, &lz_options)) return LZMA_OPTIONS_ERROR; // Allocate new buffers if needed, and do the rest of // the initialization. - if (lz_encoder_init(&next->coder->mf, allocator, &lz_options)) + if (lz_encoder_init(&coder->mf, allocator, &lz_options)) return LZMA_MEM_ERROR; // Initialize the next filter in the chain, if any. - return lzma_next_filter_init(&next->coder->next, allocator, - filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder.h b/Utilities/cmliblzma/liblzma/lz/lz_encoder.h index dcb4b2c..426dcd8 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder.h @@ -119,7 +119,7 @@ struct lzma_mf_s { lzma_action action; /// Number of elements in hash[] - uint32_t hash_size_sum; + uint32_t hash_count; /// Number of elements in son[] uint32_t sons_count; @@ -191,19 +191,18 @@ typedef struct { typedef struct { /// Data specific to the LZ-based encoder - lzma_coder *coder; + void *coder; /// Function to encode from *dict to out[] - lzma_ret (*code)(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size); + lzma_ret (*code)(void *coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); /// Free allocated resources - void (*end)(lzma_coder *coder, lzma_allocator *allocator); + void (*end)(void *coder, const lzma_allocator *allocator); /// Update the options in the middle of the encoding. - lzma_ret (*options_update)(lzma_coder *coder, - const lzma_filter *filter); + lzma_ret (*options_update)(void *coder, const lzma_filter *filter); } lzma_lz_encoder; @@ -218,7 +217,7 @@ typedef struct { /// Get pointer to the first byte not ran through the match finder -static inline uint8_t * +static inline const uint8_t * mf_ptr(const lzma_mf *mf) { return mf->buffer + mf->read_pos; @@ -296,10 +295,10 @@ mf_read(lzma_mf *mf, uint8_t *out, size_t *out_pos, size_t out_size, extern lzma_ret lzma_lz_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_encoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)); diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h index de17c54..342a333 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h @@ -39,22 +39,25 @@ // Endianness doesn't matter in hash_2_calc() (no effect on the output). #ifdef TUKLIB_FAST_UNALIGNED_ACCESS # define hash_2_calc() \ - hash_value = *(const uint16_t *)(cur) + const uint32_t hash_value = *(const uint16_t *)(cur) #else # define hash_2_calc() \ - hash_value = (uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8) + const uint32_t hash_value \ + = (uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8) #endif #define hash_3_calc() \ - temp = hash_table[cur[0]] ^ cur[1]; \ - hash_2_value = temp & HASH_2_MASK; \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8)) & mf->hash_mask + const uint32_t temp = hash_table[cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & mf->hash_mask #define hash_4_calc() \ - temp = hash_table[cur[0]] ^ cur[1]; \ - hash_2_value = temp & HASH_2_MASK; \ - hash_3_value = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ + const uint32_t temp = hash_table[cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ + const uint32_t hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ ^ (hash_table[cur[3]] << 5)) & mf->hash_mask diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c b/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c index 50c3459..7852077 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c @@ -13,6 +13,7 @@ #include "lz_encoder.h" #include "lz_encoder_hash.h" +#include "memcmplen.h" /// \brief Find matches starting from the current byte @@ -32,9 +33,8 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) if (count > 0) { #ifndef NDEBUG - uint32_t i; // Validate the matches. - for (i = 0; i < count; ++i) { + for (uint32_t i = 0; i < count; ++i) { assert(matches[i].len <= mf->nice_len); assert(matches[i].dist < mf->read_pos); assert(memcmp(mf_ptr(mf) - 1, @@ -50,9 +50,6 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) // If a match of maximum search length was found, try to // extend the match to maximum possible length. if (len_best == mf->nice_len) { - uint8_t *p1; - uint8_t *p2; - // The limit for the match length is either the // maximum match length supported by the LZ-based // encoder or the number of bytes left in the @@ -63,15 +60,13 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) // Pointer to the byte we just ran through // the match finder. - p1 = mf_ptr(mf) - 1; + const uint8_t *p1 = mf_ptr(mf) - 1; // Pointer to the beginning of the match. We need -1 // here because the match distances are zero based. - p2 = p1 - matches[count - 1].dist - 1; + const uint8_t *p2 = p1 - matches[count - 1].dist - 1; - while (len_best < limit - && p1[len_best] == p2[len_best]) - ++len_best; + len_best = lzma_memcmplen(p1, p2, len_best, limit); } } @@ -112,36 +107,35 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) static void normalize(lzma_mf *mf) { - uint32_t i; - uint32_t subvalue; - uint32_t count; - uint32_t *hash; - assert(mf->read_pos + mf->offset == MUST_NORMALIZE_POS); // In future we may not want to touch the lowest bits, because there // may be match finders that use larger resolution than one byte. - subvalue = (MUST_NORMALIZE_POS - mf->cyclic_size); + const uint32_t subvalue + = (MUST_NORMALIZE_POS - mf->cyclic_size); // & (~(UINT32_C(1) << 10) - 1); - count = mf->hash_size_sum + mf->sons_count; - hash = mf->hash; - - for (i = 0; i < count; ++i) { + for (uint32_t i = 0; i < mf->hash_count; ++i) { // If the distance is greater than the dictionary size, // we can simply mark the hash element as empty. + if (mf->hash[i] <= subvalue) + mf->hash[i] = EMPTY_HASH_VALUE; + else + mf->hash[i] -= subvalue; + } + + for (uint32_t i = 0; i < mf->sons_count; ++i) { + // Do the same for mf->son. // - // NOTE: Only the first mf->hash_size_sum elements are - // initialized for sure. There may be uninitialized elements - // in mf->son. Since we go through both mf->hash and - // mf->son here in normalization, Valgrind may complain - // that the "if" below depends on uninitialized value. In - // this case it is safe to ignore the warning. See also the - // comments in lz_encoder_init() in lz_encoder.c. - if (hash[i] <= subvalue) - hash[i] = EMPTY_HASH_VALUE; + // NOTE: There may be uninitialized elements in mf->son. + // Valgrind may complain that the "if" below depends on + // an uninitialized value. In this case it is safe to ignore + // the warning. See also the comments in lz_encoder_init() + // in lz_encoder.c. + if (mf->son[i] <= subvalue) + mf->son[i] = EMPTY_HASH_VALUE; else - hash[i] -= subvalue; + mf->son[i] -= subvalue; } // Update offset to match the new locations. @@ -204,14 +198,15 @@ move_pending(lzma_mf *mf) move_pending(mf); \ ret_op; \ } \ - cur = mf_ptr(mf); \ - pos = mf->read_pos + mf->offset + const uint8_t *cur = mf_ptr(mf); \ + const uint32_t pos = mf->read_pos + mf->offset /// Header for find functions. "return 0" indicates that zero matches /// were found. #define header_find(is_bt, len_min) \ - header(is_bt, len_min, return 0) + header(is_bt, len_min, return 0); \ + uint32_t matches_count = 0 /// Header for a loop in a skip function. "continue" tells to skip the rest @@ -268,19 +263,15 @@ hc_find_func( while (true) { const uint32_t delta = pos - cur_match; - const uint8_t *pb; if (depth-- == 0 || delta >= cyclic_size) return matches; - pb = cur - delta; + const uint8_t *const pb = cur - delta; cur_match = son[cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)]; if (pb[len_best] == cur[len_best] && pb[0] == cur[0]) { - uint32_t len = 0; - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + uint32_t len = lzma_memcmplen(pb, cur, 1, len_limit); if (len_best < len) { len_best = len; @@ -313,27 +304,21 @@ do { \ extern uint32_t lzma_mf_hc3_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t delta2, cur_match; - uint32_t len_best = 2; - uint32_t matches_count = 0; - header_find(false, 3); hash_3_calc(); - delta2 = pos - mf->hash[hash_2_value]; - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 2; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen(cur - delta2, cur, + len_best, len_limit); matches[0].len = len_best; matches[0].dist = delta2 - 1; @@ -353,22 +338,18 @@ extern void lzma_mf_hc3_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t cur_match; - if (mf_avail(mf) < 3) { move_pending(mf); continue; } - cur = mf_ptr(mf); - pos = mf->read_pos + mf->offset; + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; hash_3_calc(); - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; @@ -384,25 +365,21 @@ lzma_mf_hc3_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t delta2, delta3, cur_match; - uint32_t len_best = 1; - uint32_t matches_count = 0; - header_find(false, 4); hash_4_calc(); - delta2 = pos - mf->hash[hash_2_value]; - delta3 = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value ] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 1; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { len_best = 2; matches[0].len = 2; @@ -418,9 +395,8 @@ lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches) } if (matches_count != 0) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen(cur - delta2, cur, + len_best, len_limit); matches[matches_count - 1].len = len_best; @@ -441,22 +417,18 @@ extern void lzma_mf_hc4_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t cur_match; - if (mf_avail(mf) < 4) { move_pending(mf); continue; } - cur = mf_ptr(mf); - pos = mf->read_pos + mf->offset; + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; hash_4_calc(); - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; @@ -494,10 +466,6 @@ bt_find_func( uint32_t len1 = 0; while (true) { - uint32_t *pair; - const uint8_t *pb; - uint32_t len; - const uint32_t delta = pos - cur_match; if (depth-- == 0 || delta >= cyclic_size) { *ptr0 = EMPTY_HASH_VALUE; @@ -505,17 +473,15 @@ bt_find_func( return matches; } - pair = son + ((cyclic_pos - delta + uint32_t *const pair = son + ((cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)) << 1); - pb = cur - delta; - len = my_min(len0, len1); + const uint8_t *const pb = cur - delta; + uint32_t len = my_min(len0, len1); if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + len = lzma_memcmplen(pb, cur, len + 1, len_limit); if (len_best < len) { len_best = len; @@ -564,10 +530,6 @@ bt_skip_func( uint32_t len1 = 0; while (true) { - uint32_t *pair; - const uint8_t *pb; - uint32_t len; - const uint32_t delta = pos - cur_match; if (depth-- == 0 || delta >= cyclic_size) { *ptr0 = EMPTY_HASH_VALUE; @@ -575,16 +537,14 @@ bt_skip_func( return; } - pair = son + ((cyclic_pos - delta + uint32_t *pair = son + ((cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)) << 1); - pb = cur - delta; - len = my_min(len0, len1); + const uint8_t *pb = cur - delta; + uint32_t len = my_min(len0, len1); if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + len = lzma_memcmplen(pb, cur, len + 1, len_limit); if (len == len_limit) { *ptr1 = pair[0]; @@ -626,17 +586,11 @@ do { \ extern uint32_t lzma_mf_bt2_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t hash_value; /* hash_2_calc */ - uint32_t cur_match; - uint32_t matches_count = 0; - header_find(true, 2); hash_2_calc(); - cur_match = mf->hash[hash_value]; + const uint32_t cur_match = mf->hash[hash_value]; mf->hash[hash_value] = pos; bt_find(1); @@ -647,16 +601,11 @@ extern void lzma_mf_bt2_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t hash_value; /* hash_2_calc */ - uint32_t cur_match; - header_skip(true, 2); hash_2_calc(); - cur_match = mf->hash[hash_value]; + const uint32_t cur_match = mf->hash[hash_value]; mf->hash[hash_value] = pos; bt_skip(); @@ -670,27 +619,21 @@ lzma_mf_bt2_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_bt3_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t delta2, cur_match; - uint32_t len_best = 2; - uint32_t matches_count = 0; - header_find(true, 3); hash_3_calc(); - delta2 = pos - mf->hash[hash_2_value]; - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 2; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen( + cur, cur - delta2, len_best, len_limit); matches[0].len = len_best; matches[0].dist = delta2 - 1; @@ -710,16 +653,12 @@ extern void lzma_mf_bt3_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t cur_match; - header_skip(true, 3); hash_3_calc(); - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; @@ -735,25 +674,21 @@ lzma_mf_bt3_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t delta2, delta3, cur_match; - uint32_t len_best = 1; - uint32_t matches_count = 0; - header_find(true, 4); hash_4_calc(); - delta2 = pos - mf->hash[hash_2_value]; - delta3 = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 1; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { len_best = 2; matches[0].len = 2; @@ -769,9 +704,8 @@ lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches) } if (matches_count != 0) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen( + cur, cur - delta2, len_best, len_limit); matches[matches_count - 1].len = len_best; @@ -792,16 +726,12 @@ extern void lzma_mf_bt4_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t cur_match; - header_skip(true, 4); hash_4_calc(); - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; diff --git a/Utilities/cmliblzma/liblzma/lzma/fastpos.h b/Utilities/cmliblzma/liblzma/lzma/fastpos.h index 5a834d6..a3feea5 100644 --- a/Utilities/cmliblzma/liblzma/lzma/fastpos.h +++ b/Utilities/cmliblzma/liblzma/lzma/fastpos.h @@ -14,15 +14,15 @@ #ifndef LZMA_FASTPOS_H #define LZMA_FASTPOS_H -// LZMA encodes match distances (positions) by storing the highest two -// bits using a six-bit value [0, 63], and then the missing lower bits. -// Dictionary size is also stored using this encoding in the new .lzma +// LZMA encodes match distances by storing the highest two bits using +// a six-bit value [0, 63], and then the missing lower bits. +// Dictionary size is also stored using this encoding in the .xz // file format header. // // fastpos.h provides a way to quickly find out the correct six-bit // values. The following table gives some examples of this encoding: // -// pos return +// dist return // 0 0 // 1 1 // 2 2 @@ -48,10 +48,10 @@ // Provided functions or macros // ---------------------------- // -// get_pos_slot(pos) is the basic version. get_pos_slot_2(pos) -// assumes that pos >= FULL_DISTANCES, thus the result is at least -// FULL_DISTANCES_BITS * 2. Using get_pos_slot(pos) instead of -// get_pos_slot_2(pos) would give the same result, but get_pos_slot_2(pos) +// get_dist_slot(dist) is the basic version. get_dist_slot_2(dist) +// assumes that dist >= FULL_DISTANCES, thus the result is at least +// FULL_DISTANCES_BITS * 2. Using get_dist_slot(dist) instead of +// get_dist_slot_2(dist) would give the same result, but get_dist_slot_2(dist) // should be tiny bit faster due to the assumption being made. // // @@ -75,16 +75,15 @@ // on all systems I have tried. The size optimized version is sometimes // slightly faster, but sometimes it is a lot slower. -#include "config.h" - #ifdef HAVE_SMALL -# define get_pos_slot(pos) ((pos) <= 4 ? (pos) : get_pos_slot_2(pos)) +# define get_dist_slot(dist) \ + ((dist) <= 4 ? (dist) : get_dist_slot_2(dist)) static inline uint32_t -get_pos_slot_2(uint32_t pos) +get_dist_slot_2(uint32_t dist) { - const uint32_t i = bsr32(pos); - return (i + i) + ((pos >> (i - 1)) & 1); + const uint32_t i = bsr32(dist); + return (i + i) + ((dist >> (i - 1)) & 1); } @@ -101,39 +100,39 @@ extern const uint8_t lzma_fastpos[1 << FASTPOS_BITS]; #define fastpos_limit(extra, n) \ (UINT32_C(1) << (FASTPOS_BITS + fastpos_shift(extra, n))) -#define fastpos_result(pos, extra, n) \ - lzma_fastpos[(pos) >> fastpos_shift(extra, n)] \ +#define fastpos_result(dist, extra, n) \ + lzma_fastpos[(dist) >> fastpos_shift(extra, n)] \ + 2 * fastpos_shift(extra, n) static inline uint32_t -get_pos_slot(uint32_t pos) +get_dist_slot(uint32_t dist) { // If it is small enough, we can pick the result directly from // the precalculated table. - if (pos < fastpos_limit(0, 0)) - return lzma_fastpos[pos]; + if (dist < fastpos_limit(0, 0)) + return lzma_fastpos[dist]; - if (pos < fastpos_limit(0, 1)) - return fastpos_result(pos, 0, 1); + if (dist < fastpos_limit(0, 1)) + return fastpos_result(dist, 0, 1); - return fastpos_result(pos, 0, 2); + return fastpos_result(dist, 0, 2); } #ifdef FULL_DISTANCES_BITS static inline uint32_t -get_pos_slot_2(uint32_t pos) +get_dist_slot_2(uint32_t dist) { - assert(pos >= FULL_DISTANCES); + assert(dist >= FULL_DISTANCES); - if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 0)) - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 0); + if (dist < fastpos_limit(FULL_DISTANCES_BITS - 1, 0)) + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 0); - if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 1)) - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 1); + if (dist < fastpos_limit(FULL_DISTANCES_BITS - 1, 1)) + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 1); - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 2); + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 2); } #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c index bd2a737..878c870 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c @@ -16,7 +16,7 @@ #include "lzma_decoder.h" -struct lzma_coder_s { +typedef struct { enum sequence { SEQ_CONTROL, SEQ_UNCOMPRESSED_1, @@ -50,14 +50,16 @@ struct lzma_coder_s { bool need_dictionary_reset; lzma_options_lzma options; -}; +} lzma_lzma2_coder; static lzma_ret -lzma2_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dict, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lzma2_decode(void *coder_ptr, lzma_dict *restrict dict, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size) { + lzma_lzma2_coder *restrict coder = coder_ptr; + // With SEQ_LZMA it is possible that no new input is needed to do // some progress. The rest of the sequences assume that there is // at least one byte of input. @@ -209,8 +211,10 @@ lzma2_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dict, static void -lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +lzma2_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_lzma2_coder *coder = coder_ptr; + assert(coder->lzma.end == NULL); lzma_free(coder->lzma.coder, allocator); @@ -221,34 +225,36 @@ lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lzma2_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma2_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options) { - const lzma_options_lzma *options = opt; - - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (lz->coder == NULL) + lzma_lzma2_coder *coder = lz->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_lzma2_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + lz->coder = coder; lz->code = &lzma2_decode; lz->end = &lzma2_decoder_end; - lz->coder->lzma = LZMA_LZ_DECODER_INIT; + coder->lzma = LZMA_LZ_DECODER_INIT; } - lz->coder->sequence = SEQ_CONTROL; - lz->coder->need_properties = true; - lz->coder->need_dictionary_reset = options->preset_dict == NULL + const lzma_options_lzma *options = opt; + + coder->sequence = SEQ_CONTROL; + coder->need_properties = true; + coder->need_dictionary_reset = options->preset_dict == NULL || options->preset_dict_size == 0; - return lzma_lzma_decoder_create(&lz->coder->lzma, + return lzma_lzma_decoder_create(&coder->lzma, allocator, options, lz_options); } extern lzma_ret -lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma2_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { // LZMA2 can only be the last filter in the chain. This is enforced @@ -263,17 +269,15 @@ lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern uint64_t lzma_lzma2_decoder_memusage(const void *options) { - return sizeof(lzma_coder) + return sizeof(lzma_lzma2_coder) + lzma_lzma_decoder_memusage_nocheck(options); } extern lzma_ret -lzma_lzma2_props_decode(void **options, lzma_allocator *allocator, +lzma_lzma2_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_lzma *opt; - if (props_size != 1) return LZMA_OPTIONS_ERROR; @@ -285,7 +289,8 @@ lzma_lzma2_props_decode(void **options, lzma_allocator *allocator, if (props[0] > 40) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_lzma), allocator); + lzma_options_lzma *opt = lzma_alloc( + sizeof(lzma_options_lzma), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h index fac4ac4..ef2dcbf 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h @@ -17,12 +17,13 @@ #include "common.h" extern lzma_ret lzma_lzma2_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma2_decoder_memusage(const void *options); extern lzma_ret lzma_lzma2_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c index a3651a7..63588ee 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c @@ -17,7 +17,7 @@ #include "lzma2_encoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INIT, SEQ_LZMA_ENCODE, @@ -27,7 +27,7 @@ struct lzma_coder_s { } sequence; /// LZMA encoder - lzma_coder *lzma; + void *lzma; /// LZMA options currently in use. lzma_options_lzma opt_cur; @@ -48,20 +48,19 @@ struct lzma_coder_s { /// Buffer to hold the chunk header and LZMA compressed data uint8_t buf[LZMA2_HEADER_MAX + LZMA2_CHUNK_MAX]; -}; +} lzma_lzma2_coder; static void -lzma2_header_lzma(lzma_coder *coder) +lzma2_header_lzma(lzma_lzma2_coder *coder) { - size_t pos; - size_t size; - assert(coder->uncompressed_size > 0); assert(coder->uncompressed_size <= LZMA2_UNCOMPRESSED_MAX); assert(coder->compressed_size > 0); assert(coder->compressed_size <= LZMA2_CHUNK_MAX); + size_t pos; + if (coder->need_properties) { pos = 0; @@ -82,7 +81,7 @@ lzma2_header_lzma(lzma_coder *coder) coder->buf_pos = pos; // Uncompressed size - size = coder->uncompressed_size - 1; + size_t size = coder->uncompressed_size - 1; coder->buf[pos++] += size >> 16; coder->buf[pos++] = (size >> 8) & 0xFF; coder->buf[pos++] = size & 0xFF; @@ -109,7 +108,7 @@ lzma2_header_lzma(lzma_coder *coder) static void -lzma2_header_uncompressed(lzma_coder *coder) +lzma2_header_uncompressed(lzma_lzma2_coder *coder) { assert(coder->uncompressed_size > 0); assert(coder->uncompressed_size <= LZMA2_CHUNK_MAX); @@ -134,10 +133,12 @@ lzma2_header_uncompressed(lzma_coder *coder) static lzma_ret -lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma2_encode(void *coder_ptr, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { + lzma_lzma2_coder *restrict coder = coder_ptr; + while (*out_pos < out_size) switch (coder->sequence) { case SEQ_INIT: @@ -163,9 +164,6 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Fall through case SEQ_LZMA_ENCODE: { - uint32_t read_start; - lzma_ret ret; - // Calculate how much more uncompressed data this chunk // could accept. const uint32_t left = LZMA2_UNCOMPRESSED_MAX @@ -186,10 +184,10 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Save the start position so that we can update // coder->uncompressed_size. - read_start = mf->read_pos - mf->read_ahead; + const uint32_t read_start = mf->read_pos - mf->read_ahead; // Call the LZMA encoder until the chunk is finished. - ret = lzma_lzma_encode(coder->lzma, mf, + const lzma_ret ret = lzma_lzma_encode(coder->lzma, mf, coder->buf + LZMA2_HEADER_MAX, &coder->compressed_size, LZMA2_CHUNK_MAX, limit); @@ -266,8 +264,9 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static void -lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +lzma2_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_lzma2_coder *coder = coder_ptr; lzma_free(coder->lzma, allocator); lzma_free(coder, allocator); return; @@ -275,9 +274,9 @@ lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) +lzma2_encoder_options_update(void *coder_ptr, const lzma_filter *filter) { - lzma_options_lzma *opt; + lzma_lzma2_coder *coder = coder_ptr; // New options can be set only when there is no incomplete chunk. // This is the case at the beginning of the raw stream and right @@ -287,7 +286,7 @@ lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) // Look if there are new options. At least for now, // only lc/lp/pb can be changed. - opt = filter->options; + const lzma_options_lzma *opt = filter->options; if (coder->opt_cur.lc != opt->lc || coder->opt_cur.lp != opt->lp || coder->opt_cur.pb != opt->pb) { // Validate the options. @@ -310,36 +309,38 @@ lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) static lzma_ret -lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, +lzma2_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { if (options == NULL) return LZMA_PROG_ERROR; - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (lz->coder == NULL) + lzma_lzma2_coder *coder = lz->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_lzma2_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + lz->coder = coder; lz->code = &lzma2_encode; lz->end = &lzma2_encoder_end; lz->options_update = &lzma2_encoder_options_update; - lz->coder->lzma = NULL; + coder->lzma = NULL; } - lz->coder->opt_cur = *(const lzma_options_lzma *)(options); + coder->opt_cur = *(const lzma_options_lzma *)(options); - lz->coder->sequence = SEQ_INIT; - lz->coder->need_properties = true; - lz->coder->need_state_reset = false; - lz->coder->need_dictionary_reset - = lz->coder->opt_cur.preset_dict == NULL - || lz->coder->opt_cur.preset_dict_size == 0; + coder->sequence = SEQ_INIT; + coder->need_properties = true; + coder->need_state_reset = false; + coder->need_dictionary_reset + = coder->opt_cur.preset_dict == NULL + || coder->opt_cur.preset_dict_size == 0; // Initialize LZMA encoder - return_if_error(lzma_lzma_encoder_create(&lz->coder->lzma, allocator, - &lz->coder->opt_cur, lz_options)); + return_if_error(lzma_lzma_encoder_create(&coder->lzma, allocator, + &coder->opt_cur, lz_options)); // Make sure that we will always have enough history available in // case we need to use uncompressed chunks. They are used when the @@ -355,7 +356,7 @@ lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma2_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma2_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { return lzma_lz_encoder_init( @@ -370,7 +371,7 @@ lzma_lzma2_encoder_memusage(const void *options) if (lzma_mem == UINT64_MAX) return UINT64_MAX; - return sizeof(lzma_coder) + lzma_mem; + return sizeof(lzma_lzma2_coder) + lzma_mem; } @@ -393,7 +394,17 @@ lzma_lzma2_props_encode(const void *options, uint8_t *out) if (d == UINT32_MAX) out[0] = 40; else - out[0] = get_pos_slot(d + 1) - 24; + out[0] = get_dist_slot(d + 1) - 24; return LZMA_OK; } + + +extern uint64_t +lzma_lzma2_block_size(const void *options) +{ + const lzma_options_lzma *const opt = options; + + // Use at least 1 MiB to keep compression ratio better. + return my_max((uint64_t)(opt->dict_size) * 3, UINT64_C(1) << 20); +} diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h index ca19ef4..515f183 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h @@ -31,11 +31,13 @@ extern lzma_ret lzma_lzma2_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); extern uint64_t lzma_lzma2_encoder_memusage(const void *options); extern lzma_ret lzma_lzma2_props_encode(const void *options, uint8_t *out); +extern uint64_t lzma_lzma2_block_size(const void *options); + #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_common.h b/Utilities/cmliblzma/liblzma/lzma/lzma_common.h index 36267dc..09efd38 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_common.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_common.h @@ -129,15 +129,12 @@ static inline void literal_init(probability (*probs)[LITERAL_CODER_SIZE], uint32_t lc, uint32_t lp) { - uint32_t coders; - uint32_t i, j; - assert(lc + lp <= LZMA_LCLP_MAX); - coders = 1U << (lc + lp); + const uint32_t coders = 1U << (lc + lp); - for (i = 0; i < coders; ++i) - for (j = 0; j < LITERAL_CODER_SIZE; ++j) + for (uint32_t i = 0; i < coders; ++i) + for (uint32_t j = 0; j < LITERAL_CODER_SIZE; ++j) bit_reset(probs[i][j]); return; @@ -174,53 +171,54 @@ literal_init(probability (*probs)[LITERAL_CODER_SIZE], // Match distance // //////////////////// -// Different set of probabilities is used for match distances that have very +// Different sets of probabilities are used for match distances that have very // short match length: Lengths of 2, 3, and 4 bytes have a separate set of // probabilities for each length. The matches with longer length use a shared // set of probabilities. -#define LEN_TO_POS_STATES 4 +#define DIST_STATES 4 // Macro to get the index of the appropriate probability array. -#define get_len_to_pos_state(len) \ - ((len) < LEN_TO_POS_STATES + MATCH_LEN_MIN \ +#define get_dist_state(len) \ + ((len) < DIST_STATES + MATCH_LEN_MIN \ ? (len) - MATCH_LEN_MIN \ - : LEN_TO_POS_STATES - 1) + : DIST_STATES - 1) -// The highest two bits of a match distance (pos slot) are encoded using six -// bits. See fastpos.h for more explanation. -#define POS_SLOT_BITS 6 -#define POS_SLOTS (1 << POS_SLOT_BITS) +// The highest two bits of a match distance (distance slot) are encoded +// using six bits. See fastpos.h for more explanation. +#define DIST_SLOT_BITS 6 +#define DIST_SLOTS (1 << DIST_SLOT_BITS) // Match distances up to 127 are fully encoded using probabilities. Since -// the highest two bits (pos slot) are always encoded using six bits, the -// distances 0-3 don't need any additional bits to encode, since the pos -// slot itself is the same as the actual distance. START_POS_MODEL_INDEX -// indicates the first pos slot where at least one additional bit is needed. -#define START_POS_MODEL_INDEX 4 +// the highest two bits (distance slot) are always encoded using six bits, +// the distances 0-3 don't need any additional bits to encode, since the +// distance slot itself is the same as the actual distance. DIST_MODEL_START +// indicates the first distance slot where at least one additional bit is +// needed. +#define DIST_MODEL_START 4 // Match distances greater than 127 are encoded in three pieces: -// - pos slot: the highest two bits +// - distance slot: the highest two bits // - direct bits: 2-26 bits below the highest two bits // - alignment bits: four lowest bits // // Direct bits don't use any probabilities. // -// The pos slot value of 14 is for distances 128-191 (see the table in +// The distance slot value of 14 is for distances 128-191 (see the table in // fastpos.h to understand why). -#define END_POS_MODEL_INDEX 14 +#define DIST_MODEL_END 14 -// Pos slots that indicate a distance <= 127. -#define FULL_DISTANCES_BITS (END_POS_MODEL_INDEX / 2) +// Distance slots that indicate a distance <= 127. +#define FULL_DISTANCES_BITS (DIST_MODEL_END / 2) #define FULL_DISTANCES (1 << FULL_DISTANCES_BITS) // For match distances greater than 127, only the highest two bits and the // lowest four bits (alignment) is encoded using probabilities. #define ALIGN_BITS 4 -#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS) -#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1) +#define ALIGN_SIZE (1 << ALIGN_BITS) +#define ALIGN_MASK (ALIGN_SIZE - 1) // LZMA remembers the four most recent match distances. Reusing these distances // tends to take less space than re-encoding the actual distance value. -#define REP_DISTANCES 4 +#define REPS 4 #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c index 3c0f393..d0f29b7 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c @@ -16,6 +16,12 @@ #include "lzma_decoder.h" #include "range_decoder.h" +// The macros unroll loops with switch statements. +// Silence warnings about missing fall-through comments. +#if TUKLIB_GNUC_REQ(7, 0) +# pragma GCC diagnostic ignored "-Wimplicit-fallthrough" +#endif + #ifdef HAVE_SMALL @@ -114,33 +120,33 @@ do { \ case seq ## _CHOICE: \ rc_if_0(ld.choice, seq ## _CHOICE) { \ rc_update_0(ld.choice); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW0); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW1); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW2); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW0); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW1); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW2); \ target = symbol - LEN_LOW_SYMBOLS + MATCH_LEN_MIN; \ } else { \ rc_update_1(ld.choice); \ case seq ## _CHOICE2: \ rc_if_0(ld.choice2, seq ## _CHOICE2) { \ rc_update_0(ld.choice2); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID0); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID1); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID2); \ target = symbol - LEN_MID_SYMBOLS \ + MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \ } else { \ rc_update_1(ld.choice2); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH0); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH1); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH2); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH3); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH4); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH5); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH6); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH7); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH0); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH1); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH2); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH3); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH4); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH5); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH6); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH7); \ target = symbol - LEN_HIGH_SYMBOLS \ + MATCH_LEN_MIN \ + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ @@ -161,7 +167,7 @@ typedef struct { } lzma_length_decoder; -struct lzma_coder_s { +typedef struct { /////////////////// // Probabilities // /////////////////// @@ -193,15 +199,15 @@ struct lzma_coder_s { /// Probability tree for the highest two bits of the match distance. /// There is a separate probability tree for match lengths of /// 2 (i.e. MATCH_LEN_MIN), 3, 4, and [5, 273]. - probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; + probability dist_slot[DIST_STATES][DIST_SLOTS]; /// Probability trees for additional bits for match distance when the /// distance is in the range [4, 127]. - probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; + probability pos_special[FULL_DISTANCES - DIST_MODEL_END]; /// Probability tree for the lowest four bits of a match distance /// that is equal to or greater than 128. - probability pos_align[ALIGN_TABLE_SIZE]; + probability pos_align[ALIGN_SIZE]; /// Length of a normal match lzma_length_decoder match_len_decoder; @@ -245,8 +251,8 @@ struct lzma_coder_s { SEQ_LITERAL_WRITE, SEQ_IS_REP, seq_len(SEQ_MATCH_LEN), - seq_6(SEQ_POS_SLOT), - SEQ_POS_MODEL, + seq_6(SEQ_DIST_SLOT), + SEQ_DIST_MODEL, SEQ_DIRECT, seq_4(SEQ_ALIGN), SEQ_EOPM, @@ -277,14 +283,27 @@ struct lzma_coder_s { /// If decoding a literal: match byte. /// If decoding a match: length of the match. uint32_t len; -}; +} lzma_lzma1_decoder; static lzma_ret -lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, - const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size) +lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, + const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) { + lzma_lzma1_decoder *restrict coder = coder_ptr; + + //////////////////// + // Initialization // + //////////////////// + + { + const lzma_ret ret = rc_read_init( + &coder->rc, in, in_pos, in_size); + if (ret != LZMA_STREAM_END) + return ret; + } + /////////////// // Variables // /////////////// @@ -331,16 +350,6 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, if (no_eopm && coder->uncompressed_size < dict.limit - dict.pos) dict.limit = dict.pos + (size_t)(coder->uncompressed_size); - //////////////////// - // Initialization // - //////////////////// - - if (!rc_read_init(&coder->rc, in, in_pos, in_size)) - return LZMA_OK; - - rc = coder->rc; - rc_in_pos = *in_pos; - // The main decoder loop. The "switch" is used to restart the decoder at // correct location. Once restarted, the "switch" is no longer used. switch (coder->sequence) @@ -356,21 +365,6 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, break; rc_if_0(coder->is_match[state][pos_state], SEQ_IS_MATCH) { - static const lzma_lzma_state next_state[] = { - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_MATCH_LIT_LIT, - STATE_REP_LIT_LIT, - STATE_SHORTREP_LIT_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT, - STATE_SHORTREP_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT - }; - rc_update_0(coder->is_match[state][pos_state]); // It's a literal i.e. a single 8-bit byte. @@ -388,21 +382,16 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, rc_bit(probs[symbol], , , SEQ_LITERAL); } while (symbol < (1 << 8)); #else - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL0); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL1); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL2); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL3); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL4); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL5); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL6); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL7); + rc_bit_case(probs[symbol], , , SEQ_LITERAL0); + rc_bit_case(probs[symbol], , , SEQ_LITERAL1); + rc_bit_case(probs[symbol], , , SEQ_LITERAL2); + rc_bit_case(probs[symbol], , , SEQ_LITERAL3); + rc_bit_case(probs[symbol], , , SEQ_LITERAL4); + rc_bit_case(probs[symbol], , , SEQ_LITERAL5); + rc_bit_case(probs[symbol], , , SEQ_LITERAL6); + rc_bit_case(probs[symbol], , , SEQ_LITERAL7); #endif } else { -#ifndef HAVE_SMALL - uint32_t match_bit; - uint32_t subcoder_index; -#endif - // Decode literal with match byte. // // We store the byte we compare against @@ -441,6 +430,8 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, } while (symbol < (1 << 8)); #else // Unroll the loop. + uint32_t match_bit; + uint32_t subcoder_index; # define d(seq) \ case seq: \ @@ -474,6 +465,20 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // Use a lookup table to update to literal state, // since compared to other state updates, this would // need two branches. + static const lzma_lzma_state next_state[] = { + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_MATCH_LIT_LIT, + STATE_REP_LIT_LIT, + STATE_SHORTREP_LIT_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT, + STATE_SHORTREP_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT + }; state = next_state[state]; case SEQ_LITERAL_WRITE: @@ -509,28 +514,28 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // Prepare to decode the highest two bits of the // match distance. - probs = coder->pos_slot[get_len_to_pos_state(len)]; + probs = coder->dist_slot[get_dist_state(len)]; symbol = 1; #ifdef HAVE_SMALL - case SEQ_POS_SLOT: + case SEQ_DIST_SLOT: do { - rc_bit(probs[symbol], , , SEQ_POS_SLOT); - } while (symbol < POS_SLOTS); + rc_bit(probs[symbol], , , SEQ_DIST_SLOT); + } while (symbol < DIST_SLOTS); #else - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT0); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT1); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT2); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT3); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT4); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT5); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT0); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT1); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT2); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT3); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT4); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT5); #endif // Get rid of the highest bit that was needed for // indexing of the probability array. - symbol -= POS_SLOTS; + symbol -= DIST_SLOTS; assert(symbol <= 63); - if (symbol < START_POS_MODEL_INDEX) { + if (symbol < DIST_MODEL_START) { // Match distances [0, 3] have only two bits. rep0 = symbol; } else { @@ -540,7 +545,7 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, assert(limit >= 1 && limit <= 30); rep0 = 2 + (symbol & 1); - if (symbol < END_POS_MODEL_INDEX) { + if (symbol < DIST_MODEL_END) { // Prepare to decode the low bits for // a distance of [4, 127]. assert(limit <= 5); @@ -560,38 +565,38 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, - symbol - 1; symbol = 1; offset = 0; - case SEQ_POS_MODEL: + case SEQ_DIST_MODEL: #ifdef HAVE_SMALL do { rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); } while (++offset < limit); #else switch (limit) { case 5: assert(offset == 0); - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 4: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 3: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 2: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 1: @@ -601,9 +606,9 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // rc_bit_last() here to omit // the unneeded updating of // "symbol". - rc_bit_last(probs[symbol], 0, + rc_bit_last(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); } #endif } else { @@ -635,19 +640,19 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, } while (++offset < ALIGN_BITS); #else case SEQ_ALIGN0: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 1, SEQ_ALIGN0); case SEQ_ALIGN1: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 2, SEQ_ALIGN1); case SEQ_ALIGN2: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 4, SEQ_ALIGN2); case SEQ_ALIGN3: - // Like in SEQ_POS_MODEL, we don't + // Like in SEQ_DIST_MODEL, we don't // need "symbol" for anything else // than indexing the probability array. - rc_bit_last(coder->pos_align[symbol], 0, + rc_bit_last(coder->pos_align[symbol], , rep0 += 8, SEQ_ALIGN3); #endif @@ -732,11 +737,9 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // is stored to rep0 and rep1, rep2 and rep3 // are updated accordingly. rc_if_0(coder->is_rep1[state], SEQ_IS_REP1) { - uint32_t distance; - rc_update_0(coder->is_rep1[state]); - distance = rep1; + const uint32_t distance = rep1; rep1 = rep0; rep0 = distance; @@ -745,23 +748,19 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, case SEQ_IS_REP2: rc_if_0(coder->is_rep2[state], SEQ_IS_REP2) { - uint32_t distance; - rc_update_0(coder->is_rep2[ state]); - distance = rep2; + const uint32_t distance = rep2; rep2 = rep1; rep1 = rep0; rep0 = distance; } else { - uint32_t distance; - rc_update_1(coder->is_rep2[ state]); - distance = rep3; + const uint32_t distance = rep3; rep3 = rep2; rep2 = rep1; rep1 = rep0; @@ -849,26 +848,17 @@ out: static void -lzma_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) +lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) { + lzma_lzma1_decoder *coder = coder_ptr; coder->uncompressed_size = uncompressed_size; } -/* -extern void -lzma_lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) -{ - // This is hack. - (*(lzma_coder **)(coder))->uncompressed_size = uncompressed_size; -} -*/ static void -lzma_decoder_reset(lzma_coder *coder, const void *opt) +lzma_decoder_reset(void *coder_ptr, const void *opt) { - uint32_t i, j, pos_state; - uint32_t num_pos_states; - + lzma_lzma1_decoder *coder = coder_ptr; const lzma_options_lzma *options = opt; // NOTE: We assume that lc/lp/pb are valid since they were @@ -895,8 +885,8 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) rc_reset(coder->rc); // Bit and bittree decoders - for (i = 0; i < STATES; ++i) { - for (j = 0; j <= coder->pos_mask; ++j) { + for (uint32_t i = 0; i < STATES; ++i) { + for (uint32_t j = 0; j <= coder->pos_mask; ++j) { bit_reset(coder->is_match[i][j]); bit_reset(coder->is_rep0_long[i][j]); } @@ -907,22 +897,22 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) bit_reset(coder->is_rep2[i]); } - for (i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); + for (uint32_t i = 0; i < DIST_STATES; ++i) + bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS); - for (i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) + for (uint32_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i) bit_reset(coder->pos_special[i]); bittree_reset(coder->pos_align, ALIGN_BITS); // Len decoders (also bit/bittree) - num_pos_states = 1U << options->pb; + const uint32_t num_pos_states = 1U << options->pb; bit_reset(coder->match_len_decoder.choice); bit_reset(coder->match_len_decoder.choice2); bit_reset(coder->rep_len_decoder.choice); bit_reset(coder->rep_len_decoder.choice2); - for (pos_state = 0; pos_state < num_pos_states; ++pos_state) { + for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { bittree_reset(coder->match_len_decoder.low[pos_state], LEN_LOW_BITS); bittree_reset(coder->match_len_decoder.mid[pos_state], @@ -949,13 +939,11 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) extern lzma_ret -lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma_lzma_decoder_create(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options) { - const lzma_options_lzma *options = opt; - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); + lz->coder = lzma_alloc(sizeof(lzma_lzma1_decoder), allocator); if (lz->coder == NULL) return LZMA_MEM_ERROR; @@ -966,6 +954,7 @@ lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, // All dictionary sizes are OK here. LZ decoder will take care of // the special cases. + const lzma_options_lzma *options = opt; lz_options->dict_size = options->dict_size; lz_options->preset_dict = options->preset_dict; lz_options->preset_dict_size = options->preset_dict_size; @@ -978,7 +967,7 @@ lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, /// initialization (lzma_lzma_decoder_init() passes function pointer to /// the LZ initialization). static lzma_ret -lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { if (!is_lclppb_valid(options)) @@ -995,7 +984,7 @@ lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { // LZMA can only be the last filter in the chain. This is enforced @@ -1027,7 +1016,8 @@ extern uint64_t lzma_lzma_decoder_memusage_nocheck(const void *options) { const lzma_options_lzma *const opt = options; - return sizeof(lzma_coder) + lzma_lz_decoder_memusage(opt->dict_size); + return sizeof(lzma_lzma1_decoder) + + lzma_lz_decoder_memusage(opt->dict_size); } @@ -1042,15 +1032,14 @@ lzma_lzma_decoder_memusage(const void *options) extern lzma_ret -lzma_lzma_props_decode(void **options, lzma_allocator *allocator, +lzma_lzma_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_lzma *opt; - if (props_size != 5) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_lzma), allocator); + lzma_options_lzma *opt + = lzma_alloc(sizeof(lzma_options_lzma), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h index a463a76..fa8ecb2 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h @@ -19,12 +19,13 @@ /// Allocates and initializes LZMA decoder extern lzma_ret lzma_lzma_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma_decoder_memusage(const void *options); extern lzma_ret lzma_lzma_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); @@ -40,7 +41,7 @@ extern bool lzma_lzma_lclppb_decode( /// Allocate and setup function pointers only. This is used by LZMA1 and /// LZMA2 decoders. extern lzma_ret lzma_lzma_decoder_create( - lzma_lz_decoder *lz, lzma_allocator *allocator, + lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options); /// Gets memory usage without validating lc/lp/pb. This is used by LZMA2 diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c index e8738f4..ba9ce69 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c @@ -28,14 +28,11 @@ literal_matched(lzma_range_encoder *rc, probability *subcoder, symbol += UINT32_C(1) << 8; do { - uint32_t match_bit; - uint32_t subcoder_index; - uint32_t bit; - match_byte <<= 1; - match_bit = match_byte & offset; - subcoder_index = offset + match_bit + (symbol >> 8); - bit = (symbol >> 7) & 1; + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; rc_bit(rc, &subcoder[subcoder_index], bit); symbol <<= 1; @@ -46,7 +43,7 @@ literal_matched(lzma_range_encoder *rc, probability *subcoder, static inline void -literal(lzma_coder *coder, lzma_mf *mf, uint32_t position) +literal(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t position) { // Locate the literal byte to be encoded and the subcoder. const uint8_t cur_byte = mf->buffer[ @@ -80,19 +77,16 @@ literal(lzma_coder *coder, lzma_mf *mf, uint32_t position) static void length_update_prices(lzma_length_encoder *lc, const uint32_t pos_state) { - uint32_t a0, a1, b0, b1; - uint32_t *prices; - uint32_t i; - const uint32_t table_size = lc->table_size; lc->counters[pos_state] = table_size; - a0 = rc_bit_0_price(lc->choice); - a1 = rc_bit_1_price(lc->choice); - b0 = a1 + rc_bit_0_price(lc->choice2); - b1 = a1 + rc_bit_1_price(lc->choice2); - prices = lc->prices[pos_state]; + const uint32_t a0 = rc_bit_0_price(lc->choice); + const uint32_t a1 = rc_bit_1_price(lc->choice); + const uint32_t b0 = a1 + rc_bit_0_price(lc->choice2); + const uint32_t b1 = a1 + rc_bit_1_price(lc->choice2); + uint32_t *const prices = lc->prices[pos_state]; + uint32_t i; for (i = 0; i < table_size && i < LEN_LOW_SYMBOLS; ++i) prices[i] = a0 + rc_bittree_price(lc->low[pos_state], LEN_LOW_BITS, i); @@ -146,39 +140,36 @@ length(lzma_range_encoder *rc, lzma_length_encoder *lc, /////////// static inline void -match(lzma_coder *coder, const uint32_t pos_state, +match(lzma_lzma1_encoder *coder, const uint32_t pos_state, const uint32_t distance, const uint32_t len) { - uint32_t pos_slot; - uint32_t len_to_pos_state; - update_match(coder->state); length(&coder->rc, &coder->match_len_encoder, pos_state, len, coder->fast_mode); - pos_slot = get_pos_slot(distance); - len_to_pos_state = get_len_to_pos_state(len); - rc_bittree(&coder->rc, coder->pos_slot[len_to_pos_state], - POS_SLOT_BITS, pos_slot); + const uint32_t dist_slot = get_dist_slot(distance); + const uint32_t dist_state = get_dist_state(len); + rc_bittree(&coder->rc, coder->dist_slot[dist_state], + DIST_SLOT_BITS, dist_slot); - if (pos_slot >= START_POS_MODEL_INDEX) { - const uint32_t footer_bits = (pos_slot >> 1) - 1; - const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; - const uint32_t pos_reduced = distance - base; + if (dist_slot >= DIST_MODEL_START) { + const uint32_t footer_bits = (dist_slot >> 1) - 1; + const uint32_t base = (2 | (dist_slot & 1)) << footer_bits; + const uint32_t dist_reduced = distance - base; - if (pos_slot < END_POS_MODEL_INDEX) { - // Careful here: base - pos_slot - 1 can be -1, but + if (dist_slot < DIST_MODEL_END) { + // Careful here: base - dist_slot - 1 can be -1, but // rc_bittree_reverse starts at probs[1], not probs[0]. rc_bittree_reverse(&coder->rc, - coder->pos_special + base - pos_slot - 1, - footer_bits, pos_reduced); + coder->dist_special + base - dist_slot - 1, + footer_bits, dist_reduced); } else { - rc_direct(&coder->rc, pos_reduced >> ALIGN_BITS, + rc_direct(&coder->rc, dist_reduced >> ALIGN_BITS, footer_bits - ALIGN_BITS); rc_bittree_reverse( - &coder->rc, coder->pos_align, - ALIGN_BITS, pos_reduced & ALIGN_MASK); + &coder->rc, coder->dist_align, + ALIGN_BITS, dist_reduced & ALIGN_MASK); ++coder->align_price_count; } } @@ -196,7 +187,7 @@ match(lzma_coder *coder, const uint32_t pos_state, //////////////////// static inline void -rep_match(lzma_coder *coder, const uint32_t pos_state, +rep_match(lzma_lzma1_encoder *coder, const uint32_t pos_state, const uint32_t rep, const uint32_t len) { if (rep == 0) { @@ -240,7 +231,7 @@ rep_match(lzma_coder *coder, const uint32_t pos_state, ////////// static void -encode_symbol(lzma_coder *coder, lzma_mf *mf, +encode_symbol(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t back, uint32_t len, uint32_t position) { const uint32_t pos_state = position & coder->pos_mask; @@ -256,7 +247,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); - if (back < REP_DISTANCES) { + if (back < REPS) { // It's a repeated match i.e. the same distance // has been used earlier. rc_bit(&coder->rc, &coder->is_rep[coder->state], 1); @@ -264,7 +255,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, } else { // Normal match rc_bit(&coder->rc, &coder->is_rep[coder->state], 0); - match(coder, pos_state, back - REP_DISTANCES, len); + match(coder, pos_state, back - REPS, len); } } @@ -274,7 +265,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, static bool -encode_init(lzma_coder *coder, lzma_mf *mf) +encode_init(lzma_lzma1_encoder *coder, lzma_mf *mf) { assert(mf_position(mf) == 0); @@ -302,7 +293,7 @@ encode_init(lzma_coder *coder, lzma_mf *mf) static void -encode_eopm(lzma_coder *coder, uint32_t position) +encode_eopm(lzma_lzma1_encoder *coder, uint32_t position) { const uint32_t pos_state = position & coder->pos_mask; rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); @@ -318,23 +309,18 @@ encode_eopm(lzma_coder *coder, uint32_t position) extern lzma_ret -lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma_lzma_encode(lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, uint32_t limit) { - uint32_t position; - // Initialize the stream if no data has been encoded yet. if (!coder->is_initialized && !encode_init(coder, mf)) return LZMA_OK; // Get the lowest bits of the uncompressed offset from the LZ layer. - position = mf_position(mf); + uint32_t position = mf_position(mf); while (true) { - uint32_t len; - uint32_t back; - // Encode pending bits, if any. Calling this before encoding // the next symbol is needed only with plain LZMA, since // LZMA2 always provides big enough buffer to flush @@ -367,12 +353,14 @@ lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Get optimal match (repeat position and length). // Value ranges for pos: - // - [0, REP_DISTANCES): repeated match - // - [REP_DISTANCES, UINT32_MAX): - // match at (pos - REP_DISTANCES) + // - [0, REPS): repeated match + // - [REPS, UINT32_MAX): + // match at (pos - REPS) // - UINT32_MAX: not a match but a literal // Value ranges for len: // - [MATCH_LEN_MIN, MATCH_LEN_MAX] + uint32_t len; + uint32_t back; if (coder->fast_mode) lzma_lzma_optimum_fast(coder, mf, &back, &len); @@ -414,8 +402,8 @@ lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static lzma_ret -lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma_encode(void *coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { // Plain LZMA has no support for sync-flushing. @@ -465,12 +453,10 @@ static void length_encoder_reset(lzma_length_encoder *lencoder, const uint32_t num_pos_states, const bool fast_mode) { - size_t pos_state; - bit_reset(lencoder->choice); bit_reset(lencoder->choice2); - for (pos_state = 0; pos_state < num_pos_states; ++pos_state) { + for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS); bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS); } @@ -478,7 +464,7 @@ length_encoder_reset(lzma_length_encoder *lencoder, bittree_reset(lencoder->high, LEN_HIGH_BITS); if (!fast_mode) - for (pos_state = 0; pos_state < num_pos_states; + for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) length_update_prices(lencoder, pos_state); @@ -487,10 +473,9 @@ length_encoder_reset(lzma_length_encoder *lencoder, extern lzma_ret -lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) +lzma_lzma_encoder_reset(lzma_lzma1_encoder *coder, + const lzma_options_lzma *options) { - size_t i, j; - if (!is_options_valid(options)) return LZMA_OPTIONS_ERROR; @@ -503,14 +488,14 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) // State coder->state = STATE_LIT_LIT; - for (i = 0; i < REP_DISTANCES; ++i) + for (size_t i = 0; i < REPS; ++i) coder->reps[i] = 0; literal_init(coder->literal, options->lc, options->lp); // Bit encoders - for (i = 0; i < STATES; ++i) { - for (j = 0; j <= coder->pos_mask; ++j) { + for (size_t i = 0; i < STATES; ++i) { + for (size_t j = 0; j <= coder->pos_mask; ++j) { bit_reset(coder->is_match[i][j]); bit_reset(coder->is_rep0_long[i][j]); } @@ -521,14 +506,14 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) bit_reset(coder->is_rep2[i]); } - for (i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) - bit_reset(coder->pos_special[i]); + for (size_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i) + bit_reset(coder->dist_special[i]); // Bit tree encoders - for (i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); + for (size_t i = 0; i < DIST_STATES; ++i) + bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS); - bittree_reset(coder->pos_align, ALIGN_BITS); + bittree_reset(coder->dist_align, ALIGN_BITS); // Length encoders length_encoder_reset(&coder->match_len_encoder, @@ -561,20 +546,18 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) extern lzma_ret -lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, +lzma_lzma_encoder_create(void **coder_ptr, + const lzma_allocator *allocator, const lzma_options_lzma *options, lzma_lz_options *lz_options) { - lzma_coder *coder; - uint32_t log_size = 0; - - // Allocate lzma_coder if it wasn't already allocated. + // Allocate lzma_lzma1_encoder if it wasn't already allocated. if (*coder_ptr == NULL) { - *coder_ptr = lzma_alloc(sizeof(lzma_coder), allocator); + *coder_ptr = lzma_alloc(sizeof(lzma_lzma1_encoder), allocator); if (*coder_ptr == NULL) return LZMA_MEM_ERROR; } - coder = *coder_ptr; + lzma_lzma1_encoder *coder = *coder_ptr; // Set compression mode. We haven't validates the options yet, // but it's OK here, since nothing bad happens with invalid @@ -590,6 +573,7 @@ lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, // Set dist_table_size. // Round the dictionary size up to next 2^n. + uint32_t log_size = 0; while ((UINT32_C(1) << log_size) < options->dict_size) ++log_size; @@ -622,7 +606,7 @@ lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, static lzma_ret -lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, +lzma_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { lz->code = &lzma_encode; @@ -632,7 +616,7 @@ lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { return lzma_lz_encoder_init( @@ -643,19 +627,17 @@ lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern uint64_t lzma_lzma_encoder_memusage(const void *options) { - lzma_lz_options lz_options; - uint64_t lz_memusage; - if (!is_options_valid(options)) return UINT64_MAX; + lzma_lz_options lz_options; set_lz_options(&lz_options, options); - lz_memusage = lzma_lz_encoder_memusage(&lz_options); + const uint64_t lz_memusage = lzma_lz_encoder_memusage(&lz_options); if (lz_memusage == UINT64_MAX) return UINT64_MAX; - return (uint64_t)(sizeof(lzma_coder)) + lz_memusage; + return (uint64_t)(sizeof(lzma_lzma1_encoder)) + lz_memusage; } diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h index abb8d8b..6cfdf22 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h @@ -17,8 +17,12 @@ #include "common.h" +typedef struct lzma_lzma1_encoder_s lzma_lzma1_encoder; + + extern lzma_ret lzma_lzma_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma_encoder_memusage(const void *options); @@ -35,18 +39,18 @@ extern bool lzma_lzma_lclppb_encode( /// Initializes raw LZMA encoder; this is used by LZMA2. extern lzma_ret lzma_lzma_encoder_create( - lzma_coder **coder_ptr, lzma_allocator *allocator, + void **coder_ptr, const lzma_allocator *allocator, const lzma_options_lzma *options, lzma_lz_options *lz_options); /// Resets an already initialized LZMA encoder; this is used by LZMA2. extern lzma_ret lzma_lzma_encoder_reset( - lzma_coder *coder, const lzma_options_lzma *options); + lzma_lzma1_encoder *coder, const lzma_options_lzma *options); -extern lzma_ret lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +extern lzma_ret lzma_lzma_encode(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, uint32_t read_limit); #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c index f983126..6c53d2b 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c @@ -10,6 +10,7 @@ /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder_private.h" +#include "memcmplen.h" #define change_pair(small_dist, big_dist) \ @@ -17,17 +18,10 @@ extern void -lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res) +lzma_lzma_optimum_fast(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res) { - const uint8_t *buf; - uint32_t buf_avail; - uint32_t i; - uint32_t rep_len = 0; - uint32_t rep_index = 0; - uint32_t back_main = 0; - uint32_t limit; - const uint32_t nice_len = mf->nice_len; uint32_t len_main; @@ -40,8 +34,8 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m matches_count = coder->matches_count; } - buf = mf_ptr(mf) - 1; - buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); + const uint8_t *buf = mf_ptr(mf) - 1; + const uint32_t buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); if (buf_avail < 2) { // There's not enough input left to encode a match. @@ -51,9 +45,10 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m } // Look for repeated matches; scan the previous four match distances - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len; + uint32_t rep_len = 0; + uint32_t rep_index = 0; + for (uint32_t i = 0; i < REPS; ++i) { // Pointer to the beginning of the match candidate const uint8_t *const buf_back = buf - coder->reps[i] - 1; @@ -64,8 +59,8 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // The first two bytes matched. // Calculate the length of the match. - for (len = 2; len < buf_avail - && buf[len] == buf_back[len]; ++len) ; + const uint32_t len = lzma_memcmplen( + buf, buf_back, 2, buf_avail); // If we have found a repeated match that is at least // nice_len long, return it immediately. @@ -85,13 +80,13 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // We didn't find a long enough repeated match. Encode it as a normal // match if the match length is at least nice_len. if (len_main >= nice_len) { - *back_res = coder->matches[matches_count - 1].dist - + REP_DISTANCES; + *back_res = coder->matches[matches_count - 1].dist + REPS; *len_res = len_main; mf_skip(mf, len_main - 1); return; } + uint32_t back_main = 0; if (len_main >= 2) { back_main = coder->matches[matches_count - 1].dist; @@ -158,27 +153,17 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // the old buf pointer instead of recalculating it with mf_ptr(). ++buf; - limit = len_main - 1; - - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len; - - const uint8_t *const buf_back = buf - coder->reps[i] - 1; - - if (not_equal_16(buf, buf_back)) - continue; - - for (len = 2; len < limit - && buf[len] == buf_back[len]; ++len) ; + const uint32_t limit = my_max(2, len_main - 1); - if (len >= limit) { + for (uint32_t i = 0; i < REPS; ++i) { + if (memcmp(buf, buf - coder->reps[i] - 1, limit) == 0) { *back_res = UINT32_MAX; *len_res = 1; return; } } - *back_res = back_main + REP_DISTANCES; + *back_res = back_main + REPS; *len_res = len_main; mf_skip(mf, len_main - 2); return; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c index d3a6348..59f7734 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c @@ -11,6 +11,7 @@ #include "lzma_encoder_private.h" #include "fastpos.h" +#include "memcmplen.h" //////////// @@ -18,7 +19,7 @@ //////////// static uint32_t -get_literal_price(const lzma_coder *const coder, const uint32_t pos, +get_literal_price(const lzma_lzma1_encoder *const coder, const uint32_t pos, const uint32_t prev_byte, const bool match_mode, uint32_t match_byte, uint32_t symbol) { @@ -35,15 +36,12 @@ get_literal_price(const lzma_coder *const coder, const uint32_t pos, symbol += UINT32_C(1) << 8; do { - uint32_t match_bit; - uint32_t subcoder_index; - uint32_t bit; - match_byte <<= 1; - match_bit = match_byte & offset; - subcoder_index = offset + match_bit + (symbol >> 8); - bit = (symbol >> 7) & 1; + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; price += rc_bit_price(subcoder[subcoder_index], bit); symbol <<= 1; @@ -67,7 +65,7 @@ get_len_price(const lzma_length_encoder *const lencoder, static inline uint32_t -get_short_rep_price(const lzma_coder *const coder, +get_short_rep_price(const lzma_lzma1_encoder *const coder, const lzma_lzma_state state, const uint32_t pos_state) { return rc_bit_0_price(coder->is_rep0[state]) @@ -76,7 +74,7 @@ get_short_rep_price(const lzma_coder *const coder, static inline uint32_t -get_pure_rep_price(const lzma_coder *const coder, const uint32_t rep_index, +get_pure_rep_price(const lzma_lzma1_encoder *const coder, const uint32_t rep_index, const lzma_lzma_state state, uint32_t pos_state) { uint32_t price; @@ -101,7 +99,7 @@ get_pure_rep_price(const lzma_coder *const coder, const uint32_t rep_index, static inline uint32_t -get_rep_price(const lzma_coder *const coder, const uint32_t rep_index, +get_rep_price(const lzma_lzma1_encoder *const coder, const uint32_t rep_index, const uint32_t len, const lzma_lzma_state state, const uint32_t pos_state) { @@ -111,18 +109,18 @@ get_rep_price(const lzma_coder *const coder, const uint32_t rep_index, static inline uint32_t -get_pos_len_price(const lzma_coder *const coder, const uint32_t pos, +get_dist_len_price(const lzma_lzma1_encoder *const coder, const uint32_t dist, const uint32_t len, const uint32_t pos_state) { - const uint32_t len_to_pos_state = get_len_to_pos_state(len); + const uint32_t dist_state = get_dist_state(len); uint32_t price; - if (pos < FULL_DISTANCES) { - price = coder->distances_prices[len_to_pos_state][pos]; + if (dist < FULL_DISTANCES) { + price = coder->dist_prices[dist_state][dist]; } else { - const uint32_t pos_slot = get_pos_slot_2(pos); - price = coder->pos_slot_prices[len_to_pos_state][pos_slot] - + coder->align_prices[pos & ALIGN_MASK]; + const uint32_t dist_slot = get_dist_slot_2(dist); + price = coder->dist_slot_prices[dist_state][dist_slot] + + coder->align_prices[dist & ALIGN_MASK]; } price += get_len_price(&coder->match_len_encoder, len, pos_state); @@ -132,59 +130,53 @@ get_pos_len_price(const lzma_coder *const coder, const uint32_t pos, static void -fill_distances_prices(lzma_coder *coder) +fill_dist_prices(lzma_lzma1_encoder *coder) { - uint32_t len_to_pos_state; - uint32_t pos_slot; - uint32_t i; - - for (len_to_pos_state = 0; - len_to_pos_state < LEN_TO_POS_STATES; - ++len_to_pos_state) { + for (uint32_t dist_state = 0; dist_state < DIST_STATES; ++dist_state) { - uint32_t *const pos_slot_prices - = coder->pos_slot_prices[len_to_pos_state]; + uint32_t *const dist_slot_prices + = coder->dist_slot_prices[dist_state]; - // Price to encode the pos_slot. - for (pos_slot = 0; - pos_slot < coder->dist_table_size; ++pos_slot) - pos_slot_prices[pos_slot] = rc_bittree_price( - coder->pos_slot[len_to_pos_state], - POS_SLOT_BITS, pos_slot); + // Price to encode the dist_slot. + for (uint32_t dist_slot = 0; + dist_slot < coder->dist_table_size; ++dist_slot) + dist_slot_prices[dist_slot] = rc_bittree_price( + coder->dist_slot[dist_state], + DIST_SLOT_BITS, dist_slot); // For matches with distance >= FULL_DISTANCES, add the price // of the direct bits part of the match distance. (Align bits // are handled by fill_align_prices()). - for (pos_slot = END_POS_MODEL_INDEX; - pos_slot < coder->dist_table_size; ++pos_slot) - pos_slot_prices[pos_slot] += rc_direct_price( - ((pos_slot >> 1) - 1) - ALIGN_BITS); + for (uint32_t dist_slot = DIST_MODEL_END; + dist_slot < coder->dist_table_size; + ++dist_slot) + dist_slot_prices[dist_slot] += rc_direct_price( + ((dist_slot >> 1) - 1) - ALIGN_BITS); // Distances in the range [0, 3] are fully encoded with - // pos_slot, so they are used for coder->distances_prices + // dist_slot, so they are used for coder->dist_prices // as is. - for (i = 0; i < START_POS_MODEL_INDEX; ++i) - coder->distances_prices[len_to_pos_state][i] - = pos_slot_prices[i]; + for (uint32_t i = 0; i < DIST_MODEL_START; ++i) + coder->dist_prices[dist_state][i] + = dist_slot_prices[i]; } - // Distances in the range [4, 127] depend on pos_slot and pos_special. - // We do this in a loop separate from the above loop to avoid - // redundant calls to get_pos_slot(). - for (i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) { - const uint32_t pos_slot = get_pos_slot(i); - const uint32_t footer_bits = ((pos_slot >> 1) - 1); - const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; + // Distances in the range [4, 127] depend on dist_slot and + // dist_special. We do this in a loop separate from the above + // loop to avoid redundant calls to get_dist_slot(). + for (uint32_t i = DIST_MODEL_START; i < FULL_DISTANCES; ++i) { + const uint32_t dist_slot = get_dist_slot(i); + const uint32_t footer_bits = ((dist_slot >> 1) - 1); + const uint32_t base = (2 | (dist_slot & 1)) << footer_bits; const uint32_t price = rc_bittree_reverse_price( - coder->pos_special + base - pos_slot - 1, + coder->dist_special + base - dist_slot - 1, footer_bits, i - base); - for (len_to_pos_state = 0; - len_to_pos_state < LEN_TO_POS_STATES; - ++len_to_pos_state) - coder->distances_prices[len_to_pos_state][i] - = price + coder->pos_slot_prices[ - len_to_pos_state][pos_slot]; + for (uint32_t dist_state = 0; dist_state < DIST_STATES; + ++dist_state) + coder->dist_prices[dist_state][i] + = price + coder->dist_slot_prices[ + dist_state][dist_slot]; } coder->match_price_count = 0; @@ -193,12 +185,11 @@ fill_distances_prices(lzma_coder *coder) static void -fill_align_prices(lzma_coder *coder) +fill_align_prices(lzma_lzma1_encoder *coder) { - uint32_t i; - for (i = 0; i < ALIGN_TABLE_SIZE; ++i) + for (uint32_t i = 0; i < ALIGN_SIZE; ++i) coder->align_prices[i] = rc_bittree_reverse_price( - coder->pos_align, ALIGN_BITS, i); + coder->dist_align, ALIGN_BITS, i); coder->align_price_count = 0; return; @@ -230,18 +221,15 @@ make_short_rep(lzma_optimal *optimal) static void -backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, - uint32_t *LZMA_RESTRICT back_res, uint32_t cur) +backward(lzma_lzma1_encoder *restrict coder, uint32_t *restrict len_res, + uint32_t *restrict back_res, uint32_t cur) { + coder->opts_end_index = cur; + uint32_t pos_mem = coder->opts[cur].pos_prev; uint32_t back_mem = coder->opts[cur].back_prev; - coder->opts_end_index = cur; - do { - const uint32_t pos_prev = pos_mem; - const uint32_t back_cur = back_mem; - if (coder->opts[cur].prev_1_is_literal) { make_literal(&coder->opts[pos_mem]); coder->opts[pos_mem].pos_prev = pos_mem - 1; @@ -256,6 +244,9 @@ backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, } } + const uint32_t pos_prev = pos_mem; + const uint32_t back_cur = back_mem; + back_mem = coder->opts[pos_prev].back_prev; pos_mem = coder->opts[pos_prev].pos_prev; @@ -278,27 +269,10 @@ backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, ////////// static inline uint32_t -helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res, +helper1(lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, uint32_t position) { - uint32_t buf_avail; - const uint8_t *buf; - uint32_t rep_lens[REP_DISTANCES]; - uint32_t rep_max_index = 0; - uint32_t i; - - uint8_t current_byte; - uint8_t match_byte; - - uint32_t pos_state; - uint32_t match_price; - uint32_t rep_match_price; - uint32_t len_end; - uint32_t len; - - uint32_t normal_match_price; - const uint32_t nice_len = mf->nice_len; uint32_t len_main; @@ -312,18 +286,19 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, matches_count = coder->matches_count; } - buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); + const uint32_t buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); if (buf_avail < 2) { *back_res = UINT32_MAX; *len_res = 1; return UINT32_MAX; } - buf = mf_ptr(mf) - 1; + const uint8_t *const buf = mf_ptr(mf) - 1; - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len_test; + uint32_t rep_lens[REPS]; + uint32_t rep_max_index = 0; + for (uint32_t i = 0; i < REPS; ++i) { const uint8_t *const buf_back = buf - coder->reps[i] - 1; if (not_equal_16(buf, buf_back)) { @@ -331,12 +306,9 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, continue; } - for (len_test = 2; len_test < buf_avail - && buf[len_test] == buf_back[len_test]; - ++len_test) ; + rep_lens[i] = lzma_memcmplen(buf, buf_back, 2, buf_avail); - rep_lens[i] = len_test; - if (len_test > rep_lens[rep_max_index]) + if (rep_lens[i] > rep_lens[rep_max_index]) rep_max_index = i; } @@ -349,15 +321,14 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, if (len_main >= nice_len) { - *back_res = coder->matches[matches_count - 1].dist - + REP_DISTANCES; + *back_res = coder->matches[matches_count - 1].dist + REPS; *len_res = len_main; mf_skip(mf, len_main - 1); return UINT32_MAX; } - current_byte = *buf; - match_byte = *(buf - coder->reps[0] - 1); + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - coder->reps[0] - 1); if (len_main < 2 && current_byte != match_byte && rep_lens[rep_max_index] < 2) { @@ -368,7 +339,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, coder->opts[0].state = coder->state; - pos_state = position & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; coder->opts[1].price = rc_bit_0_price( coder->is_match[coder->state][pos_state]) @@ -378,9 +349,9 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, make_literal(&coder->opts[1]); - match_price = rc_bit_1_price( + const uint32_t match_price = rc_bit_1_price( coder->is_match[coder->state][pos_state]); - rep_match_price = match_price + const uint32_t rep_match_price = match_price + rc_bit_1_price(coder->is_rep[coder->state]); if (match_byte == current_byte) { @@ -394,7 +365,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, } } - len_end = my_max(len_main, rep_lens[rep_max_index]); + const uint32_t len_end = my_max(len_main, rep_lens[rep_max_index]); if (len_end < 2) { *back_res = coder->opts[1].back_prev; @@ -404,23 +375,21 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, coder->opts[1].pos_prev = 0; - for (i = 0; i < REP_DISTANCES; ++i) + for (uint32_t i = 0; i < REPS; ++i) coder->opts[0].backs[i] = coder->reps[i]; - len = len_end; + uint32_t len = len_end; do { coder->opts[len].price = RC_INFINITY_PRICE; } while (--len >= 2); - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t price; - + for (uint32_t i = 0; i < REPS; ++i) { uint32_t rep_len = rep_lens[i]; if (rep_len < 2) continue; - price = rep_match_price + get_pure_rep_price( + const uint32_t price = rep_match_price + get_pure_rep_price( coder, i, coder->state, pos_state); do { @@ -439,7 +408,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, } - normal_match_price = match_price + const uint32_t normal_match_price = match_price + rc_bit_0_price(coder->is_rep[coder->state]); len = rep_lens[0] >= 2 ? rep_lens[0] + 1 : 2; @@ -451,14 +420,13 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, for(; ; ++len) { const uint32_t dist = coder->matches[i].dist; const uint32_t cur_and_len_price = normal_match_price - + get_pos_len_price(coder, + + get_dist_len_price(coder, dist, len, pos_state); if (cur_and_len_price < coder->opts[len].price) { coder->opts[len].price = cur_and_len_price; coder->opts[len].pos_prev = 0; - coder->opts[len].back_prev - = dist + REP_DISTANCES; + coder->opts[len].back_prev = dist + REPS; coder->opts[len].prev_1_is_literal = false; } @@ -473,7 +441,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static inline uint32_t -helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, +helper2(lzma_lzma1_encoder *coder, uint32_t *reps, const uint8_t *buf, uint32_t len_end, uint32_t position, const uint32_t cur, const uint32_t nice_len, const uint32_t buf_avail_full) { @@ -481,19 +449,6 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, uint32_t new_len = coder->longest_match_length; uint32_t pos_prev = coder->opts[cur].pos_prev; lzma_lzma_state state; - uint32_t buf_avail; - uint32_t rep_index; - uint32_t i; - - uint32_t cur_price; - uint8_t current_byte; - uint8_t match_byte; - uint32_t pos_state; - uint32_t cur_and_1_price; - bool next_is_literal = false; - uint32_t match_price; - uint32_t rep_match_price; - uint32_t start_len = 2; if (coder->opts[cur].prev_1_is_literal) { --pos_prev; @@ -501,7 +456,7 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (coder->opts[cur].prev_2) { state = coder->opts[coder->opts[cur].pos_prev_2].state; - if (coder->opts[cur].back_prev_2 < REP_DISTANCES) + if (coder->opts[cur].back_prev_2 < REPS) update_long_rep(state); else update_match(state); @@ -530,48 +485,49 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, update_long_rep(state); } else { pos = coder->opts[cur].back_prev; - if (pos < REP_DISTANCES) + if (pos < REPS) update_long_rep(state); else update_match(state); } - if (pos < REP_DISTANCES) { - uint32_t i; - + if (pos < REPS) { reps[0] = coder->opts[pos_prev].backs[pos]; + uint32_t i; for (i = 1; i <= pos; ++i) reps[i] = coder->opts[pos_prev].backs[i - 1]; - for (; i < REP_DISTANCES; ++i) + for (; i < REPS; ++i) reps[i] = coder->opts[pos_prev].backs[i]; } else { - reps[0] = pos - REP_DISTANCES; + reps[0] = pos - REPS; - for (i = 1; i < REP_DISTANCES; ++i) + for (uint32_t i = 1; i < REPS; ++i) reps[i] = coder->opts[pos_prev].backs[i - 1]; } } coder->opts[cur].state = state; - for (i = 0; i < REP_DISTANCES; ++i) + for (uint32_t i = 0; i < REPS; ++i) coder->opts[cur].backs[i] = reps[i]; - cur_price = coder->opts[cur].price; + const uint32_t cur_price = coder->opts[cur].price; - current_byte = *buf; - match_byte = *(buf - reps[0] - 1); + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - reps[0] - 1); - pos_state = position & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; - cur_and_1_price = cur_price + const uint32_t cur_and_1_price = cur_price + rc_bit_0_price(coder->is_match[state][pos_state]) + get_literal_price(coder, position, buf[-1], !is_literal_state(state), match_byte, current_byte); + bool next_is_literal = false; + if (cur_and_1_price < coder->opts[cur + 1].price) { coder->opts[cur + 1].price = cur_and_1_price; coder->opts[cur + 1].pos_prev = cur; @@ -579,9 +535,9 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, next_is_literal = true; } - match_price = cur_price + const uint32_t match_price = cur_price + rc_bit_1_price(coder->is_match[state][pos_state]); - rep_match_price = match_price + const uint32_t rep_match_price = match_price + rc_bit_1_price(coder->is_rep[state]); if (match_byte == current_byte @@ -602,40 +558,31 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (buf_avail_full < 2) return len_end; - buf_avail = my_min(buf_avail_full, nice_len); + const uint32_t buf_avail = my_min(buf_avail_full, nice_len); if (!next_is_literal && match_byte != current_byte) { // speed optimization // try literal + rep0 const uint8_t *const buf_back = buf - reps[0] - 1; const uint32_t limit = my_min(buf_avail_full, nice_len + 1); - uint32_t len_test = 1; - while (len_test < limit && buf[len_test] == buf_back[len_test]) - ++len_test; - - --len_test; + const uint32_t len_test = lzma_memcmplen(buf, buf_back, 1, limit) - 1; if (len_test >= 2) { - uint32_t pos_state_next; - uint32_t next_rep_match_price; - uint32_t offset; - uint32_t cur_and_len_price; - lzma_lzma_state state_2 = state; update_literal(state_2); - pos_state_next = (position + 1) & coder->pos_mask; - next_rep_match_price = cur_and_1_price + const uint32_t pos_state_next = (position + 1) & coder->pos_mask; + const uint32_t next_rep_match_price = cur_and_1_price + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); //for (; len_test >= 2; --len_test) { - offset = cur + 1 + len_test; + const uint32_t offset = cur + 1 + len_test; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; - cur_and_len_price = next_rep_match_price + const uint32_t cur_and_len_price = next_rep_match_price + get_rep_price(coder, 0, len_test, state_2, pos_state_next); @@ -651,23 +598,20 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, } - for (rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) { - uint32_t len_test, len_test_2, len_test_temp; - uint32_t price, limit; + uint32_t start_len = 2; // speed optimization + for (uint32_t rep_index = 0; rep_index < REPS; ++rep_index) { const uint8_t *const buf_back = buf - reps[rep_index] - 1; if (not_equal_16(buf, buf_back)) continue; - for (len_test = 2; len_test < buf_avail - && buf[len_test] == buf_back[len_test]; - ++len_test) ; + uint32_t len_test = lzma_memcmplen(buf, buf_back, 2, buf_avail); while (len_end < cur + len_test) coder->opts[++len_end].price = RC_INFINITY_PRICE; - len_test_temp = len_test; - price = rep_match_price + get_pure_rep_price( + const uint32_t len_test_temp = len_test; + const uint32_t price = rep_match_price + get_pure_rep_price( coder, rep_index, state, pos_state); do { @@ -689,8 +633,8 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, start_len = len_test + 1; - len_test_2 = len_test + 1; - limit = my_min(buf_avail_full, + uint32_t len_test_2 = len_test + 1; + const uint32_t limit = my_min(buf_avail_full, len_test_2 + nice_len); for (; len_test_2 < limit && buf[len_test_2] == buf_back[len_test_2]; @@ -699,18 +643,12 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, len_test_2 -= len_test + 1; if (len_test_2 >= 2) { - uint32_t pos_state_next; - uint32_t cur_and_len_literal_price; - uint32_t next_rep_match_price; - uint32_t offset; - uint32_t cur_and_len_price; - lzma_lzma_state state_2 = state; update_long_rep(state_2); - pos_state_next = (position + len_test) & coder->pos_mask; + uint32_t pos_state_next = (position + len_test) & coder->pos_mask; - cur_and_len_literal_price = price + const uint32_t cur_and_len_literal_price = price + get_len_price(&coder->rep_len_encoder, len_test, pos_state) + rc_bit_0_price(coder->is_match[state_2][pos_state_next]) @@ -722,17 +660,17 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, pos_state_next = (position + len_test + 1) & coder->pos_mask; - next_rep_match_price = cur_and_len_literal_price + const uint32_t next_rep_match_price = cur_and_len_literal_price + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); //for(; len_test_2 >= 2; len_test_2--) { - offset = cur + len_test + 1 + len_test_2; + const uint32_t offset = cur + len_test + 1 + len_test_2; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; - cur_and_len_price = next_rep_match_price + const uint32_t cur_and_len_price = next_rep_match_price + get_rep_price(coder, 0, len_test_2, state_2, pos_state_next); @@ -763,29 +701,27 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (new_len >= start_len) { - uint32_t len_test; - uint32_t i = 0; - const uint32_t normal_match_price = match_price + rc_bit_0_price(coder->is_rep[state]); while (len_end < cur + new_len) coder->opts[++len_end].price = RC_INFINITY_PRICE; + uint32_t i = 0; while (start_len > coder->matches[i].len) ++i; - for (len_test = start_len; ; ++len_test) { + for (uint32_t len_test = start_len; ; ++len_test) { const uint32_t cur_back = coder->matches[i].dist; uint32_t cur_and_len_price = normal_match_price - + get_pos_len_price(coder, + + get_dist_len_price(coder, cur_back, len_test, pos_state); if (cur_and_len_price < coder->opts[cur + len_test].price) { coder->opts[cur + len_test].price = cur_and_len_price; coder->opts[cur + len_test].pos_prev = cur; coder->opts[cur + len_test].back_prev - = cur_back + REP_DISTANCES; + = cur_back + REPS; coder->opts[cur + len_test].prev_1_is_literal = false; } @@ -803,16 +739,12 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, len_test_2 -= len_test + 1; if (len_test_2 >= 2) { - uint32_t pos_state_next; - uint32_t cur_and_len_literal_price; - uint32_t next_rep_match_price; - uint32_t offset; - lzma_lzma_state state_2 = state; update_match(state_2); - pos_state_next = (position + len_test) & coder->pos_mask; + uint32_t pos_state_next + = (position + len_test) & coder->pos_mask; - cur_and_len_literal_price = cur_and_len_price + const uint32_t cur_and_len_literal_price = cur_and_len_price + rc_bit_0_price( coder->is_match[state_2][pos_state_next]) + get_literal_price(coder, @@ -825,14 +757,14 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, update_literal(state_2); pos_state_next = (pos_state_next + 1) & coder->pos_mask; - next_rep_match_price + const uint32_t next_rep_match_price = cur_and_len_literal_price + rc_bit_1_price( coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); // for(; len_test_2 >= 2; --len_test_2) { - offset = cur + len_test + 1 + len_test_2; + const uint32_t offset = cur + len_test + 1 + len_test_2; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; @@ -849,7 +781,7 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, coder->opts[offset].prev_2 = true; coder->opts[offset].pos_prev_2 = cur; coder->opts[offset].back_prev_2 - = cur_back + REP_DISTANCES; + = cur_back + REPS; } //} } @@ -865,14 +797,11 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, extern void -lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res, +lzma_lzma_optimum_normal(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, uint32_t position) { - uint32_t reps[REP_DISTANCES]; - uint32_t len_end; - uint32_t cur; - // If we have symbols pending, return the next pending symbol. if (coder->opts_end_index != coder->opts_current_index) { assert(mf->read_ahead > 0); @@ -889,9 +818,9 @@ lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT // In liblzma they were moved into this single place. if (mf->read_ahead == 0) { if (coder->match_price_count >= (1 << 7)) - fill_distances_prices(coder); + fill_dist_prices(coder); - if (coder->align_price_count >= ALIGN_TABLE_SIZE) + if (coder->align_price_count >= ALIGN_SIZE) fill_align_prices(coder); } @@ -899,13 +828,14 @@ lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT // the original function into two pieces makes it at least a little // more readable, since those two parts don't share many variables. - len_end = helper1(coder, mf, back_res, len_res, position); + uint32_t len_end = helper1(coder, mf, back_res, len_res, position); if (len_end == UINT32_MAX) return; - + uint32_t reps[REPS]; memcpy(reps, coder->reps, sizeof(reps)); + uint32_t cur; for (cur = 1; cur < len_end; ++cur) { assert(cur < OPTS); diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c index 9332abf..711df02 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c @@ -2,6 +2,7 @@ // /// \file lzma_encoder_presets.c /// \brief Encoder presets +/// \note xz needs this even when only decoding is enabled. // // Author: Lasse Collin // @@ -16,10 +17,6 @@ extern LZMA_API(lzma_bool) lzma_lzma_preset(lzma_options_lzma *options, uint32_t preset) { - static const uint8_t dict_pow2[] - = { 18, 20, 21, 22, 22, 23, 23, 24, 25, 26 }; - static const uint8_t depths[] = { 4, 8, 24, 48 }; - const uint32_t level = preset & LZMA_PRESET_LEVEL_MASK; const uint32_t flags = preset & ~LZMA_PRESET_LEVEL_MASK; const uint32_t supported_flags = LZMA_PRESET_EXTREME; @@ -34,12 +31,15 @@ lzma_lzma_preset(lzma_options_lzma *options, uint32_t preset) options->lp = LZMA_LP_DEFAULT; options->pb = LZMA_PB_DEFAULT; + static const uint8_t dict_pow2[] + = { 18, 20, 21, 22, 22, 23, 23, 24, 25, 26 }; options->dict_size = UINT32_C(1) << dict_pow2[level]; if (level <= 3) { options->mode = LZMA_MODE_FAST; options->mf = level == 0 ? LZMA_MF_HC3 : LZMA_MF_HC4; options->nice_len = level <= 1 ? 128 : 273; + static const uint8_t depths[] = { 4, 8, 24, 48 }; options->depth = depths[level]; } else { options->mode = LZMA_MODE_NORMAL; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h index 04fb29e..a2da969 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h @@ -64,12 +64,12 @@ typedef struct { uint32_t pos_prev; // pos_next; uint32_t back_prev; - uint32_t backs[REP_DISTANCES]; + uint32_t backs[REPS]; } lzma_optimal; -struct lzma_coder_s { +struct lzma_lzma1_encoder_s { /// Range encoder lzma_range_encoder rc; @@ -77,7 +77,7 @@ struct lzma_coder_s { lzma_lzma_state state; /// The four most recent match distances - uint32_t reps[REP_DISTANCES]; + uint32_t reps[REPS]; /// Array of match candidates lzma_match matches[MATCH_LEN_MAX + 1]; @@ -112,9 +112,9 @@ struct lzma_coder_s { probability is_rep1[STATES]; probability is_rep2[STATES]; probability is_rep0_long[STATES][POS_STATES_MAX]; - probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; - probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; - probability pos_align[ALIGN_TABLE_SIZE]; + probability dist_slot[DIST_STATES][DIST_SLOTS]; + probability dist_special[FULL_DISTANCES - DIST_MODEL_END]; + probability dist_align[ALIGN_SIZE]; // These are the same as in lzma_decoder.c except that the encoders // include also price tables. @@ -122,12 +122,12 @@ struct lzma_coder_s { lzma_length_encoder rep_len_encoder; // Price tables - uint32_t pos_slot_prices[LEN_TO_POS_STATES][POS_SLOTS]; - uint32_t distances_prices[LEN_TO_POS_STATES][FULL_DISTANCES]; + uint32_t dist_slot_prices[DIST_STATES][DIST_SLOTS]; + uint32_t dist_prices[DIST_STATES][FULL_DISTANCES]; uint32_t dist_table_size; uint32_t match_price_count; - uint32_t align_prices[ALIGN_TABLE_SIZE]; + uint32_t align_prices[ALIGN_SIZE]; uint32_t align_price_count; // Optimal @@ -138,11 +138,11 @@ struct lzma_coder_s { extern void lzma_lzma_optimum_fast( - lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res); + lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res); -extern void lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint32_t *LZMA_RESTRICT back_res, - uint32_t *LZMA_RESTRICT len_res, uint32_t position); +extern void lzma_lzma_optimum_normal(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, uint32_t *restrict back_res, + uint32_t *restrict len_res, uint32_t position); #endif diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_common.h b/Utilities/cmliblzma/liblzma/rangecoder/range_common.h index f15623e..2c74dc1 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_common.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_common.h @@ -40,11 +40,8 @@ // This does the same for a complete bit tree. // (A tree represented as an array.) #define bittree_reset(probs, bit_levels) \ - do { \ - uint32_t bt_i; \ - for (bt_i = 0; bt_i < (1 << (bit_levels)); ++bt_i) \ - bit_reset((probs)[bt_i]); \ - } while (0) + for (uint32_t bt_i = 0; bt_i < (1 << (bit_levels)); ++bt_i) \ + bit_reset((probs)[bt_i]) ////////////////////// diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h b/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h index 199e7b5..e0b051f 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h @@ -25,20 +25,26 @@ typedef struct { /// Reads the first five bytes to initialize the range decoder. -static inline bool -rc_read_init(lzma_range_decoder *rc, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size) +static inline lzma_ret +rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) { while (rc->init_bytes_left > 0) { if (*in_pos == in_size) - return false; + return LZMA_OK; + + // The first byte is always 0x00. It could have been omitted + // in LZMA2 but it wasn't, so one byte is wasted in every + // LZMA2 chunk. + if (rc->init_bytes_left == 5 && in[*in_pos] != 0x00) + return LZMA_DATA_ERROR; rc->code = (rc->code << 8) | in[*in_pos]; ++*in_pos; --rc->init_bytes_left; } - return true; + return LZMA_STREAM_END; } diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h b/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h index e9614f2..1e1c369 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h @@ -115,8 +115,7 @@ rc_direct(lzma_range_encoder *rc, static inline void rc_flush(lzma_range_encoder *rc) { - size_t i; - for (i = 0; i < 5; ++i) + for (size_t i = 0; i < 5; ++i) rc->symbols[rc->count++] = RC_FLUSH; } diff --git a/Utilities/cmliblzma/liblzma/simple/arm.c b/Utilities/cmliblzma/liblzma/simple/arm.c index 8dcba39..181d0e3 100644 --- a/Utilities/cmliblzma/liblzma/simple/arm.c +++ b/Utilities/cmliblzma/liblzma/simple/arm.c @@ -15,19 +15,19 @@ static size_t -arm_code(lzma_simple *simple lzma_attribute((__unused__)), +arm_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { size_t i; for (i = 0; i + 4 <= size; i += 4) { if (buffer[i + 3] == 0xEB) { - uint32_t dest; uint32_t src = (buffer[i + 2] << 16) | (buffer[i + 1] << 8) | (buffer[i + 0]); src <<= 2; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + 8 + src; else @@ -45,7 +45,7 @@ arm_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -arm_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +arm_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -54,7 +54,8 @@ arm_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_arm_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_arm_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return arm_coder_init(next, allocator, filters, true); @@ -62,7 +63,8 @@ lzma_simple_arm_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_arm_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_arm_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return arm_coder_init(next, allocator, filters, false); diff --git a/Utilities/cmliblzma/liblzma/simple/armthumb.c b/Utilities/cmliblzma/liblzma/simple/armthumb.c index 4b890a3..eab4862 100644 --- a/Utilities/cmliblzma/liblzma/simple/armthumb.c +++ b/Utilities/cmliblzma/liblzma/simple/armthumb.c @@ -15,7 +15,7 @@ static size_t -armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), +armthumb_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -23,7 +23,6 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), for (i = 0; i + 4 <= size; i += 2) { if ((buffer[i + 1] & 0xF8) == 0xF0 && (buffer[i + 3] & 0xF8) == 0xF8) { - uint32_t dest; uint32_t src = ((buffer[i + 1] & 0x7) << 19) | (buffer[i + 0] << 11) | ((buffer[i + 3] & 0x7) << 8) @@ -31,6 +30,7 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), src <<= 1; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + 4 + src; else @@ -50,7 +50,7 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -armthumb_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +armthumb_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -60,7 +60,8 @@ armthumb_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_armthumb_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return armthumb_coder_init(next, allocator, filters, true); } @@ -68,7 +69,8 @@ lzma_simple_armthumb_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return armthumb_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/ia64.c b/Utilities/cmliblzma/liblzma/simple/ia64.c index c537cac..580529e 100644 --- a/Utilities/cmliblzma/liblzma/simple/ia64.c +++ b/Utilities/cmliblzma/liblzma/simple/ia64.c @@ -15,7 +15,7 @@ static size_t -ia64_code(lzma_simple *simple lzma_attribute((__unused__)), +ia64_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -28,42 +28,36 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), size_t i; for (i = 0; i + 16 <= size; i += 16) { - size_t slot; - const uint32_t instr_template = buffer[i] & 0x1F; const uint32_t mask = BRANCH_TABLE[instr_template]; uint32_t bit_pos = 5; - for (slot = 0; slot < 3; ++slot, bit_pos += 41) { + for (size_t slot = 0; slot < 3; ++slot, bit_pos += 41) { + if (((mask >> slot) & 1) == 0) + continue; + const size_t byte_pos = (bit_pos >> 3); const uint32_t bit_res = bit_pos & 0x7; uint64_t instruction = 0; - uint64_t inst_norm; - size_t j; - - if (((mask >> slot) & 1) == 0) - continue; - for (j = 0; j < 6; ++j) + for (size_t j = 0; j < 6; ++j) instruction += (uint64_t)( buffer[i + j + byte_pos]) << (8 * j); - inst_norm = instruction >> bit_res; + uint64_t inst_norm = instruction >> bit_res; if (((inst_norm >> 37) & 0xF) == 0x5 && ((inst_norm >> 9) & 0x7) == 0 /* && (inst_norm & 0x3F)== 0 */ ) { - uint32_t dest; - size_t j; - uint32_t src = (uint32_t)( (inst_norm >> 13) & 0xFFFFF); src |= ((inst_norm >> 36) & 1) << 20; src <<= 4; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + src; else @@ -79,7 +73,7 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), instruction &= (1 << bit_res) - 1; instruction |= (inst_norm << bit_res); - for (j = 0; j < 6; j++) + for (size_t j = 0; j < 6; j++) buffer[i + j + byte_pos] = (uint8_t)( instruction >> (8 * j)); @@ -92,7 +86,7 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -ia64_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +ia64_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -102,7 +96,8 @@ ia64_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_ia64_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return ia64_coder_init(next, allocator, filters, true); } @@ -110,7 +105,8 @@ lzma_simple_ia64_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_ia64_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return ia64_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/powerpc.c b/Utilities/cmliblzma/liblzma/simple/powerpc.c index 6f83511..54dfbf1 100644 --- a/Utilities/cmliblzma/liblzma/simple/powerpc.c +++ b/Utilities/cmliblzma/liblzma/simple/powerpc.c @@ -15,7 +15,7 @@ static size_t -powerpc_code(lzma_simple *simple lzma_attribute((__unused__)), +powerpc_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -49,7 +49,7 @@ powerpc_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -powerpc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +powerpc_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -59,7 +59,8 @@ powerpc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_powerpc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return powerpc_coder_init(next, allocator, filters, true); } @@ -67,7 +68,8 @@ lzma_simple_powerpc_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_powerpc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return powerpc_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/simple_coder.c b/Utilities/cmliblzma/liblzma/simple/simple_coder.c index f3bbdd7..13ebabc 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_coder.c +++ b/Utilities/cmliblzma/liblzma/simple/simple_coder.c @@ -18,10 +18,10 @@ /// Copied or encodes/decodes more data to out[]. static lzma_ret -copy_or_code(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +copy_or_code(lzma_simple_coder *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { assert(!coder->end_was_reached); @@ -55,7 +55,7 @@ copy_or_code(lzma_coder *coder, lzma_allocator *allocator, static size_t -call_filter(lzma_coder *coder, uint8_t *buffer, size_t size) +call_filter(lzma_simple_coder *coder, uint8_t *buffer, size_t size) { const size_t filtered = coder->filter(coder->simple, coder->now_pos, coder->is_encoder, @@ -66,13 +66,12 @@ call_filter(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -simple_code(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +simple_code(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { - size_t out_avail; - size_t buf_avail; + lzma_simple_coder *coder = coder_ptr; // TODO: Add partial support for LZMA_SYNC_FLUSH. We can support it // in cases when the filter is able to filter everything. With most @@ -108,13 +107,9 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, // more data to out[] hopefully filling it completely. Then filter // the data in out[]. This step is where most of the data gets // filtered if the buffer sizes used by the application are reasonable. - out_avail = out_size - *out_pos; - buf_avail = coder->size - coder->pos; + const size_t out_avail = out_size - *out_pos; + const size_t buf_avail = coder->size - coder->pos; if (out_avail > buf_avail || buf_avail == 0) { - size_t size; - size_t filtered; - size_t unfiltered; - // Store the old position so that we know from which byte // to start filtering. const size_t out_start = *out_pos; @@ -137,10 +132,11 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, } // Filter out[]. - size = *out_pos - out_start; - filtered = call_filter(coder, out + out_start, size); + const size_t size = *out_pos - out_start; + const size_t filtered = call_filter( + coder, out + out_start, size); - unfiltered = size - filtered; + const size_t unfiltered = size - filtered; assert(unfiltered <= coder->allocated / 2); // Now we can update coder->pos and coder->size, because @@ -204,8 +200,9 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, static void -simple_coder_end(lzma_coder *coder, lzma_allocator *allocator) +simple_coder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_simple_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder->simple, allocator); lzma_free(coder, allocator); @@ -214,10 +211,12 @@ simple_coder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -simple_coder_update(lzma_coder *coder, lzma_allocator *allocator, +simple_coder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_simple_coder *coder = coder_ptr; + // No update support, just call the next filter in the chain. return lzma_next_filter_update( &coder->next, allocator, reversed_filters + 1); @@ -225,59 +224,59 @@ simple_coder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_simple_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size), size_t simple_size, size_t unfiltered_max, uint32_t alignment, bool is_encoder) { - // Allocate memory for the lzma_coder structure if needed. - if (next->coder == NULL) { + // Allocate memory for the lzma_simple_coder structure if needed. + lzma_simple_coder *coder = next->coder; + if (coder == NULL) { // Here we allocate space also for the temporary buffer. We // need twice the size of unfiltered_max, because then it // is always possible to filter at least unfiltered_max bytes // more data in coder->buffer[] if it can be filled completely. - next->coder = lzma_alloc(sizeof(lzma_coder) + coder = lzma_alloc(sizeof(lzma_simple_coder) + 2 * unfiltered_max, allocator); - if (next->coder == NULL) + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &simple_code; next->end = &simple_coder_end; next->update = &simple_coder_update; - next->coder->next = LZMA_NEXT_CODER_INIT; - next->coder->filter = filter; - next->coder->allocated = 2 * unfiltered_max; + coder->next = LZMA_NEXT_CODER_INIT; + coder->filter = filter; + coder->allocated = 2 * unfiltered_max; // Allocate memory for filter-specific data structure. if (simple_size > 0) { - next->coder->simple = lzma_alloc( - simple_size, allocator); - if (next->coder->simple == NULL) + coder->simple = lzma_alloc(simple_size, allocator); + if (coder->simple == NULL) return LZMA_MEM_ERROR; } else { - next->coder->simple = NULL; + coder->simple = NULL; } } if (filters[0].options != NULL) { const lzma_options_bcj *simple = filters[0].options; - next->coder->now_pos = simple->start_offset; - if (next->coder->now_pos & (alignment - 1)) + coder->now_pos = simple->start_offset; + if (coder->now_pos & (alignment - 1)) return LZMA_OPTIONS_ERROR; } else { - next->coder->now_pos = 0; + coder->now_pos = 0; } // Reset variables. - next->coder->is_encoder = is_encoder; - next->coder->end_was_reached = false; - next->coder->pos = 0; - next->coder->filtered = 0; - next->coder->size = 0; - - return lzma_next_filter_init( - &next->coder->next, allocator, filters + 1); + coder->is_encoder = is_encoder; + coder->end_was_reached = false; + coder->pos = 0; + coder->filtered = 0; + coder->size = 0; + + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } diff --git a/Utilities/cmliblzma/liblzma/simple/simple_coder.h b/Utilities/cmliblzma/liblzma/simple/simple_coder.h index 0952fad..19c2ee0 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_coder.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_coder.h @@ -17,44 +17,56 @@ extern lzma_ret lzma_simple_x86_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_x86_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_powerpc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_powerpc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_ia64_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_ia64_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_arm_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_arm_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_armthumb_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/simple/simple_decoder.c b/Utilities/cmliblzma/liblzma/simple/simple_decoder.c index 034e158..1d864f2 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_decoder.c +++ b/Utilities/cmliblzma/liblzma/simple/simple_decoder.c @@ -14,18 +14,17 @@ extern lzma_ret -lzma_simple_props_decode(void **options, lzma_allocator *allocator, +lzma_simple_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_bcj *opt; - if (props_size == 0) return LZMA_OK; if (props_size != 4) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_bcj), allocator); + lzma_options_bcj *opt = lzma_alloc( + sizeof(lzma_options_bcj), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/simple/simple_decoder.h b/Utilities/cmliblzma/liblzma/simple/simple_decoder.h index b8bf590..bed8d37 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_decoder.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_decoder.h @@ -16,7 +16,7 @@ #include "simple_coder.h" extern lzma_ret lzma_simple_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/simple/simple_private.h b/Utilities/cmliblzma/liblzma/simple/simple_private.h index fcf9f7c..9d2c0fd 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_private.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_private.h @@ -16,9 +16,7 @@ #include "simple_coder.h" -typedef struct lzma_simple_s lzma_simple; - -struct lzma_coder_s { +typedef struct { /// Next filter in the chain lzma_next_coder next; @@ -33,12 +31,12 @@ struct lzma_coder_s { /// Pointer to filter-specific function, which does /// the actual filtering. - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size); /// Pointer to filter-specific data, or NULL if filter doesn't need /// any extra data. - lzma_simple *simple; + void *simple; /// The lowest 32 bits of the current position in the data. Most /// filters need this to do conversions between absolute and relative @@ -62,12 +60,13 @@ struct lzma_coder_s { /// Temporary buffer uint8_t buffer[]; -}; +} lzma_simple_coder; extern lzma_ret lzma_simple_coder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters, - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + const lzma_allocator *allocator, + const lzma_filter_info *filters, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size), size_t simple_size, size_t unfiltered_max, uint32_t alignment, bool is_encoder); diff --git a/Utilities/cmliblzma/liblzma/simple/sparc.c b/Utilities/cmliblzma/liblzma/simple/sparc.c index 0ddd2ac..74b2655 100644 --- a/Utilities/cmliblzma/liblzma/simple/sparc.c +++ b/Utilities/cmliblzma/liblzma/simple/sparc.c @@ -15,7 +15,7 @@ static size_t -sparc_code(lzma_simple *simple lzma_attribute((__unused__)), +sparc_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -26,8 +26,6 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), || (buffer[i] == 0x7F && (buffer[i + 1] & 0xC0) == 0xC0)) { - uint32_t dest; - uint32_t src = ((uint32_t)buffer[i + 0] << 24) | ((uint32_t)buffer[i + 1] << 16) | ((uint32_t)buffer[i + 2] << 8) @@ -35,6 +33,7 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), src <<= 2; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + src; else @@ -58,7 +57,7 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -sparc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +sparc_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -68,7 +67,8 @@ sparc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return sparc_coder_init(next, allocator, filters, true); } @@ -76,7 +76,8 @@ lzma_simple_sparc_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return sparc_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/x86.c b/Utilities/cmliblzma/liblzma/simple/x86.c index 95858e5..23d0c42 100644 --- a/Utilities/cmliblzma/liblzma/simple/x86.c +++ b/Utilities/cmliblzma/liblzma/simple/x86.c @@ -17,14 +17,14 @@ #define Test86MSByte(b) ((b) == 0 || (b) == 0xFF) -struct lzma_simple_s { +typedef struct { uint32_t prev_mask; uint32_t prev_pos; -}; +} lzma_simple_x86; static size_t -x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, +x86_code(void *simple_ptr, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { static const bool MASK_TO_ALLOWED_STATUS[8] @@ -33,39 +33,34 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, static const uint32_t MASK_TO_BIT_NUMBER[8] = { 0, 1, 2, 2, 3, 3, 3, 3 }; + lzma_simple_x86 *simple = simple_ptr; uint32_t prev_mask = simple->prev_mask; uint32_t prev_pos = simple->prev_pos; - size_t limit; - size_t buffer_pos; - if (size < 5) return 0; if (now_pos - prev_pos > 5) prev_pos = now_pos - 5; - limit = size - 5; - buffer_pos = 0; + const size_t limit = size - 5; + size_t buffer_pos = 0; while (buffer_pos <= limit) { - uint32_t offset; - uint32_t i; - uint8_t b = buffer[buffer_pos]; if (b != 0xE8 && b != 0xE9) { ++buffer_pos; continue; } - offset = now_pos + (uint32_t)(buffer_pos) + const uint32_t offset = now_pos + (uint32_t)(buffer_pos) - prev_pos; prev_pos = now_pos + (uint32_t)(buffer_pos); if (offset > 5) { prev_mask = 0; } else { - for (i = 0; i < offset; ++i) { + for (uint32_t i = 0; i < offset; ++i) { prev_mask &= 0x77; prev_mask <<= 1; } @@ -84,8 +79,6 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, uint32_t dest; while (true) { - uint32_t i; - if (is_encoder) dest = src + (now_pos + (uint32_t)( buffer_pos) + 5); @@ -96,7 +89,8 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, if (prev_mask == 0) break; - i = MASK_TO_BIT_NUMBER[prev_mask >> 1]; + const uint32_t i = MASK_TO_BIT_NUMBER[ + prev_mask >> 1]; b = (uint8_t)(dest >> (24 - i * 8)); @@ -130,15 +124,17 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, static lzma_ret -x86_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +x86_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { const lzma_ret ret = lzma_simple_coder_init(next, allocator, filters, - &x86_code, sizeof(lzma_simple), 5, 1, is_encoder); + &x86_code, sizeof(lzma_simple_x86), 5, 1, is_encoder); if (ret == LZMA_OK) { - next->coder->simple->prev_mask = 0; - next->coder->simple->prev_pos = (uint32_t)(-5); + lzma_simple_coder *coder = next->coder; + lzma_simple_x86 *simple = coder->simple; + simple->prev_mask = 0; + simple->prev_pos = (uint32_t)(-5); } return ret; @@ -146,7 +142,8 @@ x86_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_x86_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_x86_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return x86_coder_init(next, allocator, filters, true); @@ -154,7 +151,8 @@ lzma_simple_x86_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_x86_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_x86_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return x86_coder_init(next, allocator, filters, false); diff --git a/Utilities/cmlibrhash/librhash/rhash.c b/Utilities/cmlibrhash/librhash/rhash.c index ad6249b..34e1eb3 100644 --- a/Utilities/cmlibrhash/librhash/rhash.c +++ b/Utilities/cmlibrhash/librhash/rhash.c @@ -18,6 +18,8 @@ #define _LARGEFILE64_SOURCE #define _FILE_OFFSET_BITS 64 +#include "ustd.h" /* Need this first within CMake. */ + #include <string.h> /* memset() */ #include <stdlib.h> /* free() */ #include <stddef.h> /* ptrdiff_t */ |