diff options
243 files changed, 5068 insertions, 2781 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index e20d770..1fcd8f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,7 +285,8 @@ if(CMake_RUN_IWYU) message(FATAL_ERROR "CMake_RUN_IWYU is ON but include-what-you-use is not found!") endif() set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE - "${IWYU_COMMAND};-Xiwyu;--mapping_file=${CMake_SOURCE_DIR}/Utilities/IWYU/mapping.imp;-w;-DCMAKE_IWYU") + "${IWYU_COMMAND};-Xiwyu;--mapping_file=${CMake_SOURCE_DIR}/Utilities/IWYU/mapping.imp;-w") + list(APPEND CMAKE_CXX_INCLUDE_WHAT_YOU_USE ${CMake_IWYU_OPTIONS}) endif() diff --git a/CTestCustom.cmake.in b/CTestCustom.cmake.in index eb8b630..aef375a 100644 --- a/CTestCustom.cmake.in +++ b/CTestCustom.cmake.in @@ -92,6 +92,7 @@ list(APPEND CTEST_CUSTOM_WARNING_EXCEPTION "cmFortranLexer.cxx:[0-9]+:[0-9]+: warning: Call to 'realloc' has an allocation size of 0 bytes" "testProcess.*warning: Dereference of null pointer .loaded from variable .invalidAddress.." "liblzma/simple/x86.c:[0-9]+:[0-9]+: warning: The result of the '<<' expression is undefined" + "liblzma/common/index_encoder.c:[0-9]+:[0-9]+: warning: Value stored to .* during its initialization is never read" "libuv/src/.*:[0-9]+:[0-9]+: warning: Dereference of null pointer" "libuv/src/.*:[0-9]+:[0-9]+: warning: The left operand of '==' is a garbage value" ) diff --git a/Help/command/add_link_options.rst b/Help/command/add_link_options.rst index 821198d..551d440 100644 --- a/Help/command/add_link_options.rst +++ b/Help/command/add_link_options.rst @@ -1,7 +1,7 @@ add_link_options ---------------- -Adds options to the link of targets. +Adds options to the link of shared library, module and executable targets. :: diff --git a/Help/command/ctest_submit.rst b/Help/command/ctest_submit.rst index cc9612b..a412792 100644 --- a/Help/command/ctest_submit.rst +++ b/Help/command/ctest_submit.rst @@ -65,6 +65,7 @@ Submit to CDash Upload API [HTTPHEADER <header>] [RETRY_COUNT <count>] [RETRY_DELAY <delay>] + [RETURN_VALUE <result-var>] [QUIET]) This second signature is used to upload files to CDash via the CDash @@ -73,5 +74,5 @@ with a content hash of the file. If CDash does not already have the file, then it is uploaded. Along with the file, a CDash type string is specified to tell CDash which handler to use to process the data. -This signature accepts the ``HTTPHEADER``, ``RETRY_COUNT``, ``RETRY_DELAY``, and -``QUIET`` options as described above. +This signature accepts the ``HTTPHEADER``, ``RETRY_COUNT``, ``RETRY_DELAY``, +``RETURN_VALUE`` and ``QUIET`` options as described above. diff --git a/Help/command/export.rst b/Help/command/export.rst index 0c676c6..8c49328 100644 --- a/Help/command/export.rst +++ b/Help/command/export.rst @@ -51,15 +51,15 @@ unspecified. :: - export(PACKAGE <name>) + export(PACKAGE <PackageName>) Store the current build directory in the CMake user package registry -for package ``<name>``. The find_package command may consider the -directory while searching for package ``<name>``. This helps dependent +for package ``<PackageName>``. The find_package command may consider the +directory while searching for package ``<PackageName>``. This helps dependent projects find and use a package from the current project's build tree without help from the user. Note that the entry in the package registry that this command creates works only in conjunction with a -package configuration file (``<name>Config.cmake``) that works with the +package configuration file (``<PackageName>Config.cmake``) that works with the build tree. In some cases, for example for packaging and for system wide installations, it is not desirable to write the user package registry. If the :variable:`CMAKE_EXPORT_NO_PACKAGE_REGISTRY` variable diff --git a/Help/command/find_package.rst b/Help/command/find_package.rst index a4416ab..bbe8c4d 100644 --- a/Help/command/find_package.rst +++ b/Help/command/find_package.rst @@ -5,12 +5,12 @@ Load settings for an external project. :: - find_package(<package> [version] [EXACT] [QUIET] [MODULE] + find_package(<PackageName> [version] [EXACT] [QUIET] [MODULE] [REQUIRED] [[COMPONENTS] [components...]] [OPTIONAL_COMPONENTS components...] [NO_POLICY_SCOPE]) -Finds and loads settings from an external project. ``<package>_FOUND`` +Finds and loads settings from an external project. ``<PackageName>_FOUND`` will be set to indicate whether the package was found. When the package is found package-specific information is provided through variables and :ref:`Imported Targets` documented by the package itself. The @@ -44,7 +44,7 @@ are encouraged to read on. The command has two modes by which it searches for packages: "Module" mode and "Config" mode. Module mode is available when the command is invoked with the above reduced signature. CMake searches for a file -called ``Find<package>.cmake`` in the :variable:`CMAKE_MODULE_PATH` +called ``Find<PackageName>.cmake`` in the :variable:`CMAKE_MODULE_PATH` followed by the CMake installation. If the file is found, it is read and processed by CMake. It is responsible for finding the package, checking the version, and producing any needed messages. Many @@ -54,7 +54,7 @@ option is not given the command proceeds to Config mode. The complete Config mode command signature is:: - find_package(<package> [version] [EXACT] [QUIET] + find_package(<PackageName> [version] [EXACT] [QUIET] [REQUIRED] [[COMPONENTS] [components...]] [CONFIG|NO_MODULE] [NO_POLICY_SCOPE] @@ -82,29 +82,29 @@ mode is also implied by use of options not specified in the reduced signature. Config mode attempts to locate a configuration file provided by the -package to be found. A cache entry called ``<package>_DIR`` is created to +package to be found. A cache entry called ``<PackageName>_DIR`` is created to hold the directory containing the file. By default the command -searches for a package with the name ``<package>``. If the ``NAMES`` option -is given the names following it are used instead of ``<package>``. The -command searches for a file called ``<name>Config.cmake`` or -``<lower-case-name>-config.cmake`` for each name specified. A -replacement set of possible configuration file names may be given +searches for a package with the name ``<PackageName>``. If the ``NAMES`` option +is given the names following it are used instead of ``<PackageName>``. +The command searches for a file called ``<PackageName>Config.cmake`` or +``<lower-case-package-name>-config.cmake`` for each name specified. +A replacement set of possible configuration file names may be given using the ``CONFIGS`` option. The search procedure is specified below. Once found, the configuration file is read and processed by CMake. Since the file is provided by the package it already knows the location of package contents. The full path to the configuration file -is stored in the cmake variable ``<package>_CONFIG``. +is stored in the cmake variable ``<PackageName>_CONFIG``. All configuration files which have been considered by CMake while searching for an installation of the package with an appropriate -version are stored in the cmake variable ``<package>_CONSIDERED_CONFIGS``, -the associated versions in ``<package>_CONSIDERED_VERSIONS``. +version are stored in the cmake variable ``<PackageName>_CONSIDERED_CONFIGS``, +the associated versions in ``<PackageName>_CONSIDERED_VERSIONS``. If the package configuration file cannot be found CMake will generate an error describing the problem unless the ``QUIET`` argument is specified. If ``REQUIRED`` is specified and the package is not found a fatal error is generated and the configure step stops executing. If -``<package>_DIR`` has been set to a directory not containing a +``<PackageName>_DIR`` has been set to a directory not containing a configuration file CMake will ignore it and search from scratch. When the ``[version]`` argument is given Config mode will only find a @@ -127,7 +127,7 @@ version file is loaded in a nested scope in which the following variables have been defined: ``PACKAGE_FIND_NAME`` - the ``<package>`` name + the ``<PackageName>`` ``PACKAGE_FIND_VERSION`` full requested version string ``PACKAGE_FIND_VERSION_MAJOR`` @@ -158,17 +158,17 @@ whether the configuration file provides an acceptable version. They are not available after the find_package call returns. If the version is acceptable the following variables are set: -``<package>_VERSION`` +``<PackageName>_VERSION`` full provided version string -``<package>_VERSION_MAJOR`` +``<PackageName>_VERSION_MAJOR`` major version if provided, else 0 -``<package>_VERSION_MINOR`` +``<PackageName>_VERSION_MINOR`` minor version if provided, else 0 -``<package>_VERSION_PATCH`` +``<PackageName>_VERSION_PATCH`` patch version if provided, else 0 -``<package>_VERSION_TWEAK`` +``<PackageName>_VERSION_TWEAK`` tweak version if provided, else 0 -``<package>_VERSION_COUNT`` +``<PackageName>_VERSION_COUNT`` number of version components, 0 to 4 and the corresponding package configuration file is loaded. @@ -192,7 +192,7 @@ Much of the interface is provided for completeness and for use internally by find-modules loaded by Module mode. Most user code should simply call:: - find_package(<package> [major[.minor]] [EXACT] [REQUIRED|QUIET]) + find_package(<PackageName> [major[.minor]] [EXACT] [REQUIRED|QUIET]) in order to find a package. Package maintainers providing CMake package configuration files are encouraged to name and install them @@ -228,7 +228,7 @@ containing a configuration file:: <prefix>/<name>.app/Contents/Resources/CMake/ (A) In all cases the ``<name>`` is treated as case-insensitive and corresponds -to any of the names specified (``<package>`` or names given by ``NAMES``). +to any of the names specified (``<PackageName>`` or names given by ``NAMES``). Paths with ``lib/<arch>`` are enabled if the :variable:`CMAKE_LIBRARY_ARCHITECTURE` variable is set. ``lib*`` includes one @@ -286,7 +286,7 @@ enabled. (``;`` on Windows and ``:`` on UNIX). This can be skipped if ``NO_CMAKE_ENVIRONMENT_PATH`` is passed:: - <package>_DIR + <PackageName>_DIR CMAKE_PREFIX_PATH CMAKE_FRAMEWORK_PATH CMAKE_APPBUNDLE_PATH @@ -329,7 +329,7 @@ enabled. hard-coded guesses. .. |FIND_XXX| replace:: find_package -.. |FIND_ARGS_XXX| replace:: <package> +.. |FIND_ARGS_XXX| replace:: <PackageName> .. |CMAKE_FIND_ROOT_PATH_MODE_XXX| replace:: :variable:`CMAKE_FIND_ROOT_PATH_MODE_PACKAGE` @@ -344,28 +344,28 @@ defines variables to provide information about the call arguments (and restores their original state before returning): ``CMAKE_FIND_PACKAGE_NAME`` - the ``<package>`` name which is searched for -``<package>_FIND_REQUIRED`` + the ``<PackageName>`` which is searched for +``<PackageName>_FIND_REQUIRED`` true if ``REQUIRED`` option was given -``<package>_FIND_QUIETLY`` +``<PackageName>_FIND_QUIETLY`` true if ``QUIET`` option was given -``<package>_FIND_VERSION`` +``<PackageName>_FIND_VERSION`` full requested version string -``<package>_FIND_VERSION_MAJOR`` +``<PackageName>_FIND_VERSION_MAJOR`` major version if requested, else 0 -``<package>_FIND_VERSION_MINOR`` +``<PackageName>_FIND_VERSION_MINOR`` minor version if requested, else 0 -``<package>_FIND_VERSION_PATCH`` +``<PackageName>_FIND_VERSION_PATCH`` patch version if requested, else 0 -``<package>_FIND_VERSION_TWEAK`` +``<PackageName>_FIND_VERSION_TWEAK`` tweak version if requested, else 0 -``<package>_FIND_VERSION_COUNT`` +``<PackageName>_FIND_VERSION_COUNT`` number of version components, 0 to 4 -``<package>_FIND_VERSION_EXACT`` +``<PackageName>_FIND_VERSION_EXACT`` true if ``EXACT`` option was given -``<package>_FIND_COMPONENTS`` +``<PackageName>_FIND_COMPONENTS`` list of requested components -``<package>_FIND_REQUIRED_<c>`` +``<PackageName>_FIND_REQUIRED_<c>`` true if component ``<c>`` is required, false if component ``<c>`` is optional @@ -375,7 +375,7 @@ In Config mode ``find_package`` handles ``REQUIRED``, ``QUIET``, and ``[version]`` options automatically but leaves it to the package configuration file to handle components in a way that makes sense for the package. The package configuration file may set -``<package>_FOUND`` to false to tell ``find_package`` that component +``<PackageName>_FOUND`` to false to tell ``find_package`` that component requirements are not satisfied. See the :command:`cmake_policy` command documentation for discussion diff --git a/Help/command/unset.rst b/Help/command/unset.rst index a1fc95c..c19dd31 100644 --- a/Help/command/unset.rst +++ b/Help/command/unset.rst @@ -7,9 +7,16 @@ Unset a variable, cache variable, or environment variable. unset(<variable> [CACHE | PARENT_SCOPE]) -Removes the specified variable causing it to become undefined. If -``CACHE`` is present then the variable is removed from the cache instead -of the current scope. +Removes a normal variable from the current scope, causing it +to become undefined. If ``CACHE`` is present, then a cache variable +is removed instead of a normal variable. Note that when evaluating +:ref:`Variable References` of the form ``${VAR}``, CMake first searches +for a normal variable with that name. If no such normal variable exists, +CMake will then search for a cache entry with that name. Because of this +unsetting a normal variable can expose a cache variable that was previously +hidden. To force a variable reference of the form ``${VAR}`` to return an +empty string, use ``set(<variable> "")``, which clears the normal variable +but leaves it defined. If ``PARENT_SCOPE`` is present then the variable is removed from the scope above the current scope. See the same option in the :command:`set` command diff --git a/Help/cpack_gen/wix.rst b/Help/cpack_gen/wix.rst index 3822fa0..fc8a098 100644 --- a/Help/cpack_gen/wix.rst +++ b/Help/cpack_gen/wix.rst @@ -214,8 +214,8 @@ Windows using WiX. If this variable is set the generated installer will create an entry in the windows registry key - ``HKEY_LOCAL_MACHINE\Software\Kitware\CMake\Packages\<package>`` - The value for ``<package>`` is provided by this variable. + ``HKEY_LOCAL_MACHINE\Software\Kitware\CMake\Packages\<PackageName>`` + The value for ``<PackageName>`` is provided by this variable. Assuming you also install a CMake configuration file this will allow other CMake projects to find your package with diff --git a/Help/generator/Visual Studio 10 2010.rst b/Help/generator/Visual Studio 10 2010.rst index 04eef10..b8d6872 100644 --- a/Help/generator/Visual Studio 10 2010.rst +++ b/Help/generator/Visual Studio 10 2010.rst @@ -18,6 +18,12 @@ a target platform name optionally at the end of this generator name: For compatibility with CMake versions prior to 3.0, one may specify this generator using the name ``Visual Studio 10`` without the year component. +Project Types +^^^^^^^^^^^^^ + +Only Visual C++ and C# projects may be generated. Other types of +projects (Database, Website, etc.) are not supported. + Toolset Selection ^^^^^^^^^^^^^^^^^ diff --git a/Help/generator/Visual Studio 11 2012.rst b/Help/generator/Visual Studio 11 2012.rst index 347a153..8e9998a 100644 --- a/Help/generator/Visual Studio 11 2012.rst +++ b/Help/generator/Visual Studio 11 2012.rst @@ -21,6 +21,12 @@ a target platform name optionally at the end of this generator name: For compatibility with CMake versions prior to 3.0, one may specify this generator using the name "Visual Studio 11" without the year component. +Project Types +^^^^^^^^^^^^^ + +Only Visual C++ and C# projects may be generated. Other types of +projects (JavaScript, Database, Website, etc.) are not supported. + Toolset Selection ^^^^^^^^^^^^^^^^^ diff --git a/Help/generator/Visual Studio 12 2013.rst b/Help/generator/Visual Studio 12 2013.rst index 5071b20..03f7586 100644 --- a/Help/generator/Visual Studio 12 2013.rst +++ b/Help/generator/Visual Studio 12 2013.rst @@ -18,6 +18,12 @@ a target platform name optionally at the end of this generator name: For compatibility with CMake versions prior to 3.0, one may specify this generator using the name "Visual Studio 12" without the year component. +Project Types +^^^^^^^^^^^^^ + +Only Visual C++ and C# projects may be generated. Other types of +projects (JavaScript, Powershell, Python, etc.) are not supported. + Toolset Selection ^^^^^^^^^^^^^^^^^ diff --git a/Help/generator/Visual Studio 14 2015.rst b/Help/generator/Visual Studio 14 2015.rst index 64254b5..e55bc73 100644 --- a/Help/generator/Visual Studio 14 2015.rst +++ b/Help/generator/Visual Studio 14 2015.rst @@ -15,6 +15,12 @@ a target platform name optionally at the end of this generator name: ``Visual Studio 14 2015 ARM`` Specify target platform ``ARM``. +Project Types +^^^^^^^^^^^^^ + +Only Visual C++ and C# projects may be generated. Other types of +projects (JavaScript, Powershell, Python, etc.) are not supported. + Toolset Selection ^^^^^^^^^^^^^^^^^ diff --git a/Help/generator/Visual Studio 15 2017.rst b/Help/generator/Visual Studio 15 2017.rst index 2cf1aa0..809be4b 100644 --- a/Help/generator/Visual Studio 15 2017.rst +++ b/Help/generator/Visual Studio 15 2017.rst @@ -15,6 +15,12 @@ a target platform name optionally at the end of this generator name: ``Visual Studio 15 2017 ARM`` Specify target platform ``ARM``. +Project Types +^^^^^^^^^^^^^ + +Only Visual C++ and C# projects may be generated. Other types of +projects (JavaScript, Powershell, Python, etc.) are not supported. + Instance Selection ^^^^^^^^^^^^^^^^^^ diff --git a/Help/manual/cmake-developer.7.rst b/Help/manual/cmake-developer.7.rst index 32e8cfc..f05c4b1 100644 --- a/Help/manual/cmake-developer.7.rst +++ b/Help/manual/cmake-developer.7.rst @@ -579,11 +579,11 @@ file and the ``Help/manual/cmake-modules.7.rst`` toctree entry. Find Modules ------------ -A "find module" is a ``Modules/Find<package>.cmake`` file to be loaded -by the :command:`find_package` command when invoked for ``<package>``. +A "find module" is a ``Modules/Find<PackageName>.cmake`` file to be loaded +by the :command:`find_package` command when invoked for ``<PackageName>``. The primary task of a find module is to determine whether a package -exists on the system, set the ``<package>_FOUND`` variable to reflect +exists on the system, set the ``<PackageName>_FOUND`` variable to reflect this and provide any variables, macros and imported targets required to use the package. A find module is useful in cases where an upstream library does not provide a diff --git a/Help/manual/cmake-language.7.rst b/Help/manual/cmake-language.7.rst index 87f8f9d..591f73d 100644 --- a/Help/manual/cmake-language.7.rst +++ b/Help/manual/cmake-language.7.rst @@ -399,6 +399,11 @@ and how their values are set. An *environment variable reference* has the form ``$ENV{VAR}`` and is evaluated in the same contexts as a normal variable reference. +See :variable:`ENV` for more information. + +A *cache variable reference* has the form ``$CACHE{VAR}`` and +is evaluated in the same contexts as a normal variable reference. +See :variable:`CACHE` for more information. Comments -------- @@ -543,6 +548,8 @@ to the binding in the current directory scope, if any. If a is found, or no binding is found, CMake then searches for a cache entry. If a cache entry is found, its value is used. Otherwise, the variable reference evaluates to an empty string. +The ``$CACHE{VAR}`` syntax can be used to do direct cache entry +lookups. The :manual:`cmake-variables(7)` manual documents many variables that are provided by CMake or have meaning to CMake when set diff --git a/Help/manual/cmake-packages.7.rst b/Help/manual/cmake-packages.7.rst index c9442bc..876ca84 100644 --- a/Help/manual/cmake-packages.7.rst +++ b/Help/manual/cmake-packages.7.rst @@ -73,7 +73,7 @@ Handling of ``COMPONENTS`` and ``OPTIONAL_COMPONENTS`` is defined by the package. By setting the :variable:`CMAKE_DISABLE_FIND_PACKAGE_<PackageName>` variable to -``TRUE``, the ``PackageName`` package will not be searched, and will always +``TRUE``, the ``<PackageName>`` package will not be searched, and will always be ``NOTFOUND``. .. _`Config File Packages`: @@ -92,9 +92,9 @@ packages, that is, they belong with the header files and any other files provided to assist downstreams in using the package. A set of variables which provide package status information are also set -automatically when using a config-file package. The ``<Package>_FOUND`` +automatically when using a config-file package. The ``<PackageName>_FOUND`` variable is set to true or false, depending on whether the package was -found. The ``<Package>_DIR`` cache variable is set to the location of the +found. The ``<PackageName>_DIR`` cache variable is set to the location of the package configuration file. Find-module Packages @@ -108,10 +108,10 @@ file, it is not shipped with upstream, but is used by downstream to find the files by guessing locations of files with platform-specific hints. Unlike the case of an upstream-provided package configuration file, no single point -of reference identifies the package as being found, so the ``<Package>_FOUND`` +of reference identifies the package as being found, so the ``<PackageName>_FOUND`` variable is not automatically set by the :command:`find_package` command. It can still be expected to be set by convention however and should be set by -the author of the Find-module. Similarly there is no ``<Package>_DIR`` variable, +the author of the Find-module. Similarly there is no ``<PackageName>_DIR`` variable, but each of the artifacts such as library locations and header file locations provide a separate cache variable. @@ -197,7 +197,7 @@ When the :command:`find_package` command loads a version file it first sets the following variables: ``PACKAGE_FIND_NAME`` - The <package> name + The ``<PackageName>`` ``PACKAGE_FIND_VERSION`` Full requested version string @@ -240,26 +240,26 @@ variables. When the version file claims to be an acceptable match for the requested version the find_package command sets the following variables for use by the project: -``<package>_VERSION`` +``<PackageName>_VERSION`` Full provided version string -``<package>_VERSION_MAJOR`` +``<PackageName>_VERSION_MAJOR`` Major version if provided, else 0 -``<package>_VERSION_MINOR`` +``<PackageName>_VERSION_MINOR`` Minor version if provided, else 0 -``<package>_VERSION_PATCH`` +``<PackageName>_VERSION_PATCH`` Patch version if provided, else 0 -``<package>_VERSION_TWEAK`` +``<PackageName>_VERSION_TWEAK`` Tweak version if provided, else 0 -``<package>_VERSION_COUNT`` +``<PackageName>_VERSION_COUNT`` Number of version components, 0 to 4 The variables report the version of the package that was actually found. -The ``<package>`` part of their name matches the argument given to the +The ``<PackageName>`` part of their name matches the argument given to the :command:`find_package` command. .. _`Creating Packages`: @@ -347,8 +347,8 @@ The :module:`CMakePackageConfigHelpers` module provides a macro for creating a simple ``ConfigVersion.cmake`` file. This file sets the version of the package. It is read by CMake when :command:`find_package` is called to determine the compatibility with the requested version, and to set some -version-specific variables ``<Package>_VERSION``, ``<Package>_VERSION_MAJOR``, -``<Package>_VERSION_MINOR`` etc. The :command:`install(EXPORT)` command is +version-specific variables ``<PackageName>_VERSION``, ``<PackageName>_VERSION_MAJOR``, +``<PackageName>_VERSION_MINOR`` etc. The :command:`install(EXPORT)` command is used to export the targets in the ``ClimbingStatsTargets`` export-set, defined previously by the :command:`install(TARGETS)` command. This command generates the ``ClimbingStatsTargets.cmake`` file to contain :prop_tgt:`IMPORTED` @@ -432,8 +432,8 @@ the dependency is not found, along with a diagnostic that the ``ClimbingStats`` package can not be used without the ``Stats`` package. If ``COMPONENTS`` are specified when the downstream uses :command:`find_package`, -they are listed in the ``<Package>_FIND_COMPONENTS`` variable. If a particular -component is non-optional, then the ``<Package>_FIND_REQUIRED_<comp>`` will +they are listed in the ``<PackageName>_FIND_COMPONENTS`` variable. If a particular +component is non-optional, then the ``<PackageName>_FIND_REQUIRED_<comp>`` will be true. This can be tested with logic in the package configuration file: .. code-block:: cmake @@ -580,8 +580,8 @@ non-standard install locations or directly in their own build trees. A project may populate either the user or system registry (using its own means, see below) to refer to its location. In either case the package should store at the registered location a -`Package Configuration File`_ (``<package>Config.cmake``) and optionally a -`Package Version File`_ (``<package>ConfigVersion.cmake``). +`Package Configuration File`_ (``<PackageName>Config.cmake``) and optionally a +`Package Version File`_ (``<PackageName>ConfigVersion.cmake``). The :command:`find_package` command searches the two package registries as two of the search steps specified in its documentation. If it has @@ -603,18 +603,18 @@ must be manually taught to register their packages if desired. On Windows the user package registry is stored in the Windows registry under a key in ``HKEY_CURRENT_USER``. -A ``<package>`` may appear under registry key:: +A ``<PackageName>`` may appear under registry key:: - HKEY_CURRENT_USER\Software\Kitware\CMake\Packages\<package> + HKEY_CURRENT_USER\Software\Kitware\CMake\Packages\<PackageName> as a ``REG_SZ`` value, with arbitrary name, that specifies the directory containing the package configuration file. On UNIX platforms the user package registry is stored in the user home -directory under ``~/.cmake/packages``. A ``<package>`` may appear under +directory under ``~/.cmake/packages``. A ``<PackageName>`` may appear under the directory:: - ~/.cmake/packages/<package> + ~/.cmake/packages/<PackageName> as a file, with arbitrary name, whose content specifies the directory containing the package configuration file. @@ -629,10 +629,10 @@ CMake currently provides no interface to add to the system package registry. Installers must be manually taught to register their packages if desired. On Windows the system package registry is stored in the Windows registry -under a key in ``HKEY_LOCAL_MACHINE``. A ``<package>`` may appear under +under a key in ``HKEY_LOCAL_MACHINE``. A ``<PackageName>`` may appear under registry key:: - HKEY_LOCAL_MACHINE\Software\Kitware\CMake\Packages\<package> + HKEY_LOCAL_MACHINE\Software\Kitware\CMake\Packages\<PackageName> as a ``REG_SZ`` value, with arbitrary name, that specifies the directory containing the package configuration file. diff --git a/Help/manual/cmake-variables.7.rst b/Help/manual/cmake-variables.7.rst index 5a3cd81..4ddbfe5 100644 --- a/Help/manual/cmake-variables.7.rst +++ b/Help/manual/cmake-variables.7.rst @@ -230,6 +230,7 @@ Variables that Describe the System /variable/ANDROID /variable/APPLE /variable/BORLAND + /variable/CACHE /variable/CMAKE_CL_64 /variable/CMAKE_COMPILER_2005 /variable/CMAKE_HOST_APPLE diff --git a/Help/prop_dir/LINK_OPTIONS.rst b/Help/prop_dir/LINK_OPTIONS.rst index 04b9e08..15c555f 100644 --- a/Help/prop_dir/LINK_OPTIONS.rst +++ b/Help/prop_dir/LINK_OPTIONS.rst @@ -1,7 +1,8 @@ LINK_OPTIONS ------------ -List of options to use for the link step. +List of options to use for the link step of shared library, module +and executable targets. This property holds a :ref:`;-list <CMake Language Lists>` of options given so far to the :command:`add_link_options` command. diff --git a/Help/prop_tgt/LINK_FLAGS.rst b/Help/prop_tgt/LINK_FLAGS.rst index e0b72b5..3a344e6 100644 --- a/Help/prop_tgt/LINK_FLAGS.rst +++ b/Help/prop_tgt/LINK_FLAGS.rst @@ -1,7 +1,9 @@ LINK_FLAGS ---------- -Additional flags to use when linking this target. +Additional flags to use when linking this target if it is a shared library, +module library, or an executable. Static libraries need to use +:prop_tgt:`STATIC_LIBRARY_FLAGS`. The LINK_FLAGS property, managed as a string, can be used to add extra flags to the link step of a target. :prop_tgt:`LINK_FLAGS_<CONFIG>` will add to the diff --git a/Help/prop_tgt/LINK_FLAGS_CONFIG.rst b/Help/prop_tgt/LINK_FLAGS_CONFIG.rst index 1f2910b..c30ecf1 100644 --- a/Help/prop_tgt/LINK_FLAGS_CONFIG.rst +++ b/Help/prop_tgt/LINK_FLAGS_CONFIG.rst @@ -1,9 +1,9 @@ LINK_FLAGS_<CONFIG> ------------------- -Per-configuration linker flags for a target. +Per-configuration linker flags for a shared library, module or executable target. -This is the configuration-specific version of LINK_FLAGS. +This is the configuration-specific version of :prop_tgt:`LINK_FLAGS`. .. note:: diff --git a/Help/prop_tgt/STATIC_LIBRARY_FLAGS.rst b/Help/prop_tgt/STATIC_LIBRARY_FLAGS.rst index d3b2cd4..c91b1ae 100644 --- a/Help/prop_tgt/STATIC_LIBRARY_FLAGS.rst +++ b/Help/prop_tgt/STATIC_LIBRARY_FLAGS.rst @@ -1,6 +1,12 @@ STATIC_LIBRARY_FLAGS -------------------- -Extra flags to use when linking static libraries. +Archiver (or MSVC librarian) flags for a static library target. +Targets that are shared libraries, modules, or executables can use +the :prop_tgt:`LINK_OPTIONS` or :prop_tgt:`LINK_FLAGS` target property. -Extra flags to use when linking a static library. +The STATIC_LIBRARY_FLAGS property, managed as a string, can be used to add +extra flags to the link step of a static library target. +:prop_tgt:`STATIC_LIBRARY_FLAGS_<CONFIG>` will add to the configuration +``<CONFIG>``, for example, ``DEBUG``, ``RELEASE``, ``MINSIZEREL``, +``RELWITHDEBINFO``, ... diff --git a/Help/prop_tgt/STATIC_LIBRARY_FLAGS_CONFIG.rst b/Help/prop_tgt/STATIC_LIBRARY_FLAGS_CONFIG.rst index cca353d..b867ac4 100644 --- a/Help/prop_tgt/STATIC_LIBRARY_FLAGS_CONFIG.rst +++ b/Help/prop_tgt/STATIC_LIBRARY_FLAGS_CONFIG.rst @@ -1,6 +1,6 @@ STATIC_LIBRARY_FLAGS_<CONFIG> ----------------------------- -Per-configuration flags for creating a static library. +Per-configuration archiver (or MSVC librarian) flags for a static library target. -This is the configuration-specific version of STATIC_LIBRARY_FLAGS. +This is the configuration-specific version of :prop_tgt:`STATIC_LIBRARY_FLAGS`. diff --git a/Help/variable/CACHE.rst b/Help/variable/CACHE.rst new file mode 100644 index 0000000..230739a --- /dev/null +++ b/Help/variable/CACHE.rst @@ -0,0 +1,17 @@ +CACHE +----- + +Read cache variables. + +Use the syntax ``$CACHE{VAR}`` to read cache entry ``VAR``. +See the :ref:`cmake-language(7) variables <CMake Language Variables>` +documentation for more complete documentation of the interaction of +normal variables and cache entries. + +When evaluating :ref:`Variable References` of the form ``${VAR}``, +CMake first searches for a normal variable with that name, and if not +found CMake will search for a cache entry with that name. +The ``$CACHE{VAR}`` syntax can be used to do direct cache lookup and +ignore any existing normal variable. + +See the :command:`set` command to see how to write cache variables. diff --git a/Help/variable/CMAKE_LANG_CLANG_TIDY.rst b/Help/variable/CMAKE_LANG_CLANG_TIDY.rst index d1fccbb..bd49de3 100644 --- a/Help/variable/CMAKE_LANG_CLANG_TIDY.rst +++ b/Help/variable/CMAKE_LANG_CLANG_TIDY.rst @@ -9,5 +9,5 @@ created. For example: .. code-block:: cmake - set(CMAKE_CXX_CLANG_TIDY clang-tidy checks=-*,readability-*) + set(CMAKE_CXX_CLANG_TIDY clang-tidy -checks=-*,readability-*) add_executable(foo foo.cxx) diff --git a/Help/variable/ENV.rst b/Help/variable/ENV.rst index 368152a..98677dd 100644 --- a/Help/variable/ENV.rst +++ b/Help/variable/ENV.rst @@ -1,7 +1,8 @@ ENV --- -Access environment variables. +Read environment variables. -Use the syntax ``$ENV{VAR}`` to read environment variable ``VAR``. See also -the :command:`set` command to set ``ENV{VAR}``. +Use the syntax ``$ENV{VAR}`` to read environment variable ``VAR``. + +See the :command:`set` command to see how to write environment variables. diff --git a/Modules/CMakeCUDAInformation.cmake b/Modules/CMakeCUDAInformation.cmake index 8e62941..43ae989 100644 --- a/Modules/CMakeCUDAInformation.cmake +++ b/Modules/CMakeCUDAInformation.cmake @@ -185,7 +185,7 @@ if(__IMPLICT_DLINK_DIRS) endif() set(__IMPLICT_DLINK_FLAGS ) foreach(dir ${__IMPLICT_DLINK_DIRS}) - if(EXISTS "${dir}/libcublas_device.a") + if(EXISTS "${dir}/libcurand_static.a") string(APPEND __IMPLICT_DLINK_FLAGS " -L\"${dir}\"") endif() endforeach() diff --git a/Modules/CMakeDetermineCompilerId.cmake b/Modules/CMakeDetermineCompilerId.cmake index 16dde65..dc4d9be 100644 --- a/Modules/CMakeDetermineCompilerId.cmake +++ b/Modules/CMakeDetermineCompilerId.cmake @@ -355,6 +355,12 @@ Id flags: ${testflags} ${CMAKE_${lang}_COMPILER_ID_FLAGS_ALWAYS} else() set(id_development_team "") endif() + if(DEFINED CMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY) + set(id_code_sign_identity + "CODE_SIGN_IDENTITY = \"${CMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY}\";") + else() + set(id_code_sign_identity "") + endif() configure_file(${CMAKE_ROOT}/Modules/CompilerId/Xcode-3.pbxproj.in ${id_dir}/CompilerId${lang}.xcodeproj/project.pbxproj @ONLY) unset(_ENV_MACOSX_DEPLOYMENT_TARGET) diff --git a/Modules/CMakeExtraGeneratorDetermineCompilerMacrosAndIncludeDirs.cmake b/Modules/CMakeExtraGeneratorDetermineCompilerMacrosAndIncludeDirs.cmake index 3a72622..55e0373 100644 --- a/Modules/CMakeExtraGeneratorDetermineCompilerMacrosAndIncludeDirs.cmake +++ b/Modules/CMakeExtraGeneratorDetermineCompilerMacrosAndIncludeDirs.cmake @@ -66,10 +66,10 @@ macro(_DETERMINE_GCC_SYSTEM_INCLUDE_DIRS _lang _resultIncludeDirs _resultDefines #message(STATUS "m1: -${CMAKE_MATCH_1}- m2: -${CMAKE_MATCH_2}- m3: -${CMAKE_MATCH_3}-") list(APPEND ${_resultDefines} "${_name}") - if(_value) - list(APPEND ${_resultDefines} "${_value}") - else() + if ("${_value}" STREQUAL "") list(APPEND ${_resultDefines} " ") + else() + list(APPEND ${_resultDefines} "${_value}") endif() endforeach() diff --git a/Modules/CMakeFindDependencyMacro.cmake b/Modules/CMakeFindDependencyMacro.cmake index 6a89fff..de1a332 100644 --- a/Modules/CMakeFindDependencyMacro.cmake +++ b/Modules/CMakeFindDependencyMacro.cmake @@ -14,7 +14,7 @@ CMakeFindDependencyMacro It is designed to be used in a :ref:`Package Configuration File <Config File Packages>` - (``<package>Config.cmake``). ``find_dependency`` forwards the correct + (``<PackageName>Config.cmake``). ``find_dependency`` forwards the correct parameters for ``QUIET`` and ``REQUIRED`` which were passed to the original :command:`find_package` call. Any additional arguments specified are forwarded to :command:`find_package`. diff --git a/Modules/CMakePackageConfigHelpers.cmake b/Modules/CMakePackageConfigHelpers.cmake index 790d408..d5301d7 100644 --- a/Modules/CMakePackageConfigHelpers.cmake +++ b/Modules/CMakePackageConfigHelpers.cmake @@ -27,10 +27,10 @@ # ) # # ``configure_package_config_file()`` should be used instead of the plain -# :command:`configure_file()` command when creating the ``<Name>Config.cmake`` -# or ``<Name>-config.cmake`` file for installing a project or library. It helps -# making the resulting package relocatable by avoiding hardcoded paths in the -# installed ``Config.cmake`` file. +# :command:`configure_file()` command when creating the ``<PackageName>Config.cmake`` +# or ``<PackageName>-config.cmake`` file for installing a project or library. +# It helps making the resulting package relocatable by avoiding hardcoded paths +# in the installed ``Config.cmake`` file. # # In a ``FooConfig.cmake`` file there may be code like this to make the install # destinations know to the using project: @@ -101,7 +101,7 @@ # When using the ``NO_SET_AND_CHECK_MACRO``, this macro is not generated # into the ``FooConfig.cmake`` file. # -# ``check_required_components(<package_name>)`` should be called at the end of +# ``check_required_components(<PackageName>)`` should be called at the end of # the ``FooConfig.cmake`` file. This macro checks whether all requested, # non-optional components have been found, and if this is not the case, sets # the ``Foo_FOUND`` variable to ``FALSE``, so that the package is considered to @@ -127,7 +127,7 @@ # COMPATIBILITY <AnyNewerVersion|SameMajorVersion|SameMinorVersion|ExactVersion> ) # # -# Writes a file for use as ``<package>ConfigVersion.cmake`` file to +# Writes a file for use as ``<PackageName>ConfigVersion.cmake`` file to # ``<filename>``. See the documentation of :command:`find_package()` for # details on this. # diff --git a/Modules/CompilerId/Xcode-3.pbxproj.in b/Modules/CompilerId/Xcode-3.pbxproj.in index 94bcbf8..4686b64 100644 --- a/Modules/CompilerId/Xcode-3.pbxproj.in +++ b/Modules/CompilerId/Xcode-3.pbxproj.in @@ -73,6 +73,7 @@ isa = XCBuildConfiguration; buildSettings = { @id_development_team@ + @id_code_sign_identity@ PRODUCT_NAME = CompilerId@id_lang@; }; name = Debug; diff --git a/Modules/FindCUDA.cmake b/Modules/FindCUDA.cmake index 21cace3..1650e55 100644 --- a/Modules/FindCUDA.cmake +++ b/Modules/FindCUDA.cmake @@ -971,7 +971,8 @@ if(NOT CUDA_VERSION VERSION_LESS "3.2") find_cuda_helper_libs(nvcuvid) endif() endif() -if(CUDA_VERSION VERSION_GREATER "5.0") +if(CUDA_VERSION VERSION_GREATER "5.0" AND CUDA_VERSION VERSION_LESS "9.2") + # In CUDA 9.2 cublas_device was deprecated find_cuda_helper_libs(cublas_device) endif() diff --git a/Modules/FindPackageHandleStandardArgs.cmake b/Modules/FindPackageHandleStandardArgs.cmake index 67f6bd6..1722d6a 100644 --- a/Modules/FindPackageHandleStandardArgs.cmake +++ b/Modules/FindPackageHandleStandardArgs.cmake @@ -145,7 +145,7 @@ endmacro() # internal helper macro to generate the failure message when used in CONFIG_MODE: macro(_FPHSA_HANDLE_FAILURE_CONFIG_MODE) - # <name>_CONFIG is set, but FOUND is false, this means that some other of the REQUIRED_VARS was not found: + # <PackageName>_CONFIG is set, but FOUND is false, this means that some other of the REQUIRED_VARS was not found: if(${_NAME}_CONFIG) _FPHSA_FAILURE_MESSAGE("${FPHSA_FAIL_MESSAGE}: missing:${MISSING_VARS} (found ${${_NAME}_CONFIG} ${VERSION_MSG})") else() @@ -199,7 +199,7 @@ function(FIND_PACKAGE_HANDLE_STANDARD_ARGS _NAME _FIRST_ARG) set(FPHSA_FAIL_MESSAGE "DEFAULT_MSG") endif() - # In config-mode, we rely on the variable <package>_CONFIG, which is set by find_package() + # In config-mode, we rely on the variable <PackageName>_CONFIG, which is set by find_package() # when it successfully found the config-file, including version checking: if(FPHSA_CONFIG_MODE) list(INSERT FPHSA_REQUIRED_VARS 0 ${_NAME}_CONFIG) diff --git a/Modules/Platform/Windows-Intel.cmake b/Modules/Platform/Windows-Intel.cmake index ba31001..96b1760 100644 --- a/Modules/Platform/Windows-Intel.cmake +++ b/Modules/Platform/Windows-Intel.cmake @@ -11,7 +11,7 @@ set(__WINDOWS_INTEL 1) include(Platform/Windows-MSVC) macro(__windows_compiler_intel lang) __windows_compiler_msvc(${lang}) - string(REPLACE "<CMAKE_LINKER> /lib" "lib" CMAKE_${lang}_CREATE_STATIC_LIBRARY "${CMAKE_${lang}_CREATE_STATIC_LIBRARY}") + string(REPLACE "<CMAKE_LINKER> /lib" "xilib" CMAKE_${lang}_CREATE_STATIC_LIBRARY "${CMAKE_${lang}_CREATE_STATIC_LIBRARY}") foreach(rule CREATE_SHARED_LIBRARY CREATE_SHARED_MODULE LINK_EXECUTABLE) string(REPLACE "<CMAKE_LINKER>" "xilink" CMAKE_${lang}_${rule} "${CMAKE_${lang}_${rule}}") endforeach() diff --git a/Modules/Platform/Windows-NVIDIA-CUDA.cmake b/Modules/Platform/Windows-NVIDIA-CUDA.cmake index f1c1f2d..ba1638f 100644 --- a/Modules/Platform/Windows-NVIDIA-CUDA.cmake +++ b/Modules/Platform/Windows-NVIDIA-CUDA.cmake @@ -44,7 +44,7 @@ if(__IMPLICT_DLINK_DIRS) endif() set(__IMPLICT_DLINK_FLAGS ) foreach(dir ${__IMPLICT_DLINK_DIRS}) - if(EXISTS "${dir}/cublas_device.lib") + if(EXISTS "${dir}/curand_static.lib") string(APPEND __IMPLICT_DLINK_FLAGS " -L\"${dir}\"") endif() endforeach() diff --git a/Source/CMakeVersion.cmake b/Source/CMakeVersion.cmake index 8b5f738..dc64c6c 100644 --- a/Source/CMakeVersion.cmake +++ b/Source/CMakeVersion.cmake @@ -1,5 +1,5 @@ # CMake version number components. set(CMake_VERSION_MAJOR 3) set(CMake_VERSION_MINOR 12) -set(CMake_VERSION_PATCH 20180808) +set(CMake_VERSION_PATCH 20180827) #set(CMake_VERSION_RC 1) diff --git a/Source/CPack/IFW/cmCPackIFWGenerator.cxx b/Source/CPack/IFW/cmCPackIFWGenerator.cxx index cf8334a..180c92e 100644 --- a/Source/CPack/IFW/cmCPackIFWGenerator.cxx +++ b/Source/CPack/IFW/cmCPackIFWGenerator.cxx @@ -90,7 +90,7 @@ int cmCPackIFWGenerator::PackageFiles() ifwCmd.c_str(), &output, &output, &retVal, nullptr, this->GeneratorVerbose, cmDuration::zero()); if (!res || retVal) { - cmGeneratedFileStream ofs(ifwTmpFile.c_str()); + cmGeneratedFileStream ofs(ifwTmpFile); ofs << "# Run command: " << ifwCmd << std::endl << "# Output:" << std::endl << output << std::endl; @@ -203,7 +203,7 @@ int cmCPackIFWGenerator::PackageFiles() ifwCmd.c_str(), &output, &output, &retVal, nullptr, this->GeneratorVerbose, cmDuration::zero()); if (!res || retVal) { - cmGeneratedFileStream ofs(ifwTmpFile.c_str()); + cmGeneratedFileStream ofs(ifwTmpFile); ofs << "# Run command: " << ifwCmd << std::endl << "# Output:" << std::endl << output << std::endl; diff --git a/Source/CPack/IFW/cmCPackIFWInstaller.cxx b/Source/CPack/IFW/cmCPackIFWInstaller.cxx index 36cf08c..8f492af 100644 --- a/Source/CPack/IFW/cmCPackIFWInstaller.cxx +++ b/Source/CPack/IFW/cmCPackIFWInstaller.cxx @@ -306,7 +306,7 @@ void cmCPackIFWInstaller::GenerateInstallerFile() } // Output stream - cmGeneratedFileStream fout((this->Directory + "/config/config.xml").data()); + cmGeneratedFileStream fout(this->Directory + "/config/config.xml"); cmXMLWriter xout(fout); xout.StartDocument(); diff --git a/Source/CPack/IFW/cmCPackIFWPackage.cxx b/Source/CPack/IFW/cmCPackIFWPackage.cxx index f24ab69..67e279c 100644 --- a/Source/CPack/IFW/cmCPackIFWPackage.cxx +++ b/Source/CPack/IFW/cmCPackIFWPackage.cxx @@ -528,7 +528,7 @@ void cmCPackIFWPackage::GeneratePackageFile() } // Output stream - cmGeneratedFileStream fout((this->Directory + "/meta/package.xml").data()); + cmGeneratedFileStream fout(this->Directory + "/meta/package.xml"); cmXMLWriter xout(fout); xout.StartDocument(); diff --git a/Source/CPack/IFW/cmCPackIFWRepository.cxx b/Source/CPack/IFW/cmCPackIFWRepository.cxx index a01fc4e..987cad8 100644 --- a/Source/CPack/IFW/cmCPackIFWRepository.cxx +++ b/Source/CPack/IFW/cmCPackIFWRepository.cxx @@ -183,7 +183,7 @@ bool cmCPackIFWRepository::PatchUpdatesXml() this->Directory + "/repository/UpdatesPatch.xml"; // Output stream - cmGeneratedFileStream fout(updatesPatchXml.data()); + cmGeneratedFileStream fout(updatesPatchXml); cmXMLWriter xout(fout); xout.StartDocument(); diff --git a/Source/CPack/cmCPackArchiveGenerator.cxx b/Source/CPack/cmCPackArchiveGenerator.cxx index 593b4bf..ee1070f 100644 --- a/Source/CPack/cmCPackArchiveGenerator.cxx +++ b/Source/CPack/cmCPackArchiveGenerator.cxx @@ -103,7 +103,7 @@ int cmCPackArchiveGenerator::addOneComponentToArchive( */ #define DECLARE_AND_OPEN_ARCHIVE(filename, archive) \ cmGeneratedFileStream gf; \ - gf.Open((filename).c_str(), false, true); \ + gf.Open((filename), false, true); \ if (!GenerateHeader(&gf)) { \ cmCPackLogger(cmCPackLog::LOG_ERROR, \ "Problem to generate Header for archive <" \ diff --git a/Source/CPack/cmCPackDebGenerator.cxx b/Source/CPack/cmCPackDebGenerator.cxx index c53dd32..972fe6f 100644 --- a/Source/CPack/cmCPackDebGenerator.cxx +++ b/Source/CPack/cmCPackDebGenerator.cxx @@ -238,7 +238,7 @@ int cmCPackDebGenerator::createDeb() const std::string strGenWDIR(this->GetOption("GEN_WDIR")); const std::string dbfilename = strGenWDIR + "/debian-binary"; { // the scope is needed for cmGeneratedFileStream - cmGeneratedFileStream out(dbfilename.c_str()); + cmGeneratedFileStream out(dbfilename); out << "2.0"; out << std::endl; // required for valid debian package } @@ -287,7 +287,7 @@ int cmCPackDebGenerator::createDeb() this->GetOption("GEN_CPACK_DEBIAN_PACKAGE_SOURCE"); { // the scope is needed for cmGeneratedFileStream - cmGeneratedFileStream out(ctlfilename.c_str()); + cmGeneratedFileStream out(ctlfilename); out << "Package: " << debian_pkg_name << "\n"; out << "Version: " << debian_pkg_version << "\n"; out << "Section: " << debian_pkg_section << "\n"; @@ -347,7 +347,7 @@ int cmCPackDebGenerator::createDeb() const bool gen_shibs = this->IsOn("CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS") && debian_pkg_shlibs && *debian_pkg_shlibs; if (gen_shibs) { - cmGeneratedFileStream out(shlibsfilename.c_str()); + cmGeneratedFileStream out(shlibsfilename); out << debian_pkg_shlibs; out << std::endl; } @@ -355,7 +355,7 @@ int cmCPackDebGenerator::createDeb() const std::string postinst = strGenWDIR + "/postinst"; const std::string postrm = strGenWDIR + "/postrm"; if (this->IsOn("GEN_CPACK_DEBIAN_GENERATE_POSTINST")) { - cmGeneratedFileStream out(postinst.c_str()); + cmGeneratedFileStream out(postinst); out << "#!/bin/sh\n\n" "set -e\n\n" "if [ \"$1\" = \"configure\" ]; then\n" @@ -363,7 +363,7 @@ int cmCPackDebGenerator::createDeb() "fi\n"; } if (this->IsOn("GEN_CPACK_DEBIAN_GENERATE_POSTRM")) { - cmGeneratedFileStream out(postrm.c_str()); + cmGeneratedFileStream out(postrm); out << "#!/bin/sh\n\n" "set -e\n\n" "if [ \"$1\" = \"remove\" ]; then\n" @@ -412,7 +412,7 @@ int cmCPackDebGenerator::createDeb() // atomic file generation for data.tar { cmGeneratedFileStream fileStream_data_tar; - fileStream_data_tar.Open(filename_data_tar.c_str(), false, true); + fileStream_data_tar.Open(filename_data_tar, false, true); if (!fileStream_data_tar) { cmCPackLogger(cmCPackLog::LOG_ERROR, "Error opening the file \"" @@ -493,7 +493,7 @@ int cmCPackDebGenerator::createDeb() std::string md5filename = strGenWDIR + "/md5sums"; { // the scope is needed for cmGeneratedFileStream - cmGeneratedFileStream out(md5filename.c_str()); + cmGeneratedFileStream out(md5filename); std::string topLevelWithTrailingSlash = this->GetOption("CPACK_TEMPORARY_DIRECTORY"); @@ -528,7 +528,7 @@ int cmCPackDebGenerator::createDeb() // atomic file generation for control.tar { cmGeneratedFileStream fileStream_control_tar; - fileStream_control_tar.Open(filename_control_tar.c_str(), false, true); + fileStream_control_tar.Open(filename_control_tar, false, true); if (!fileStream_control_tar) { cmCPackLogger(cmCPackLog::LOG_ERROR, "Error opening the file \"" << filename_control_tar @@ -662,7 +662,7 @@ int cmCPackDebGenerator::createDeb() std::string const outputPath = outputDir + "/" + outputName; std::string const tlDir = strGenWDIR + "/"; cmGeneratedFileStream debStream; - debStream.Open(outputPath.c_str(), false, true); + debStream.Open(outputPath, false, true); cmArchiveWrite deb(debStream, cmArchiveWrite::CompressNone, "arbsd"); // uid/gid should be the one of the root user, and this root user has diff --git a/Source/CPack/cmCPackGenerator.cxx b/Source/CPack/cmCPackGenerator.cxx index 7014676..6347eed 100644 --- a/Source/CPack/cmCPackGenerator.cxx +++ b/Source/CPack/cmCPackGenerator.cxx @@ -288,7 +288,7 @@ int cmCPackGenerator::InstallProjectViaInstallCommands( if (!resB || retVal) { std::string tmpFile = this->GetOption("CPACK_TOPLEVEL_DIRECTORY"); tmpFile += "/InstallOutput.log"; - cmGeneratedFileStream ofs(tmpFile.c_str()); + cmGeneratedFileStream ofs(tmpFile); ofs << "# Run command: " << ic << std::endl << "# Output:" << std::endl << output << std::endl; @@ -369,7 +369,7 @@ int cmCPackGenerator::InstallProjectViaInstalledDirectories( inFile += '/'; } for (cmsys::RegularExpression& reg : ignoreFilesRegex) { - if (reg.find(inFile.c_str())) { + if (reg.find(inFile)) { cmCPackLogger(cmCPackLog::LOG_VERBOSE, "Ignore file: " << inFile << std::endl); skip = true; @@ -660,7 +660,7 @@ int cmCPackGenerator::RunPreinstallTarget( if (!resB || retVal) { std::string tmpFile = this->GetOption("CPACK_TOPLEVEL_DIRECTORY"); tmpFile += "/PreinstallOutput.log"; - cmGeneratedFileStream ofs(tmpFile.c_str()); + cmGeneratedFileStream ofs(tmpFile); ofs << "# Run command: " << buildCommand << std::endl << "# Directory: " << installDirectory << std::endl << "# Output:" << std::endl diff --git a/Source/CPack/cmCPackNSISGenerator.cxx b/Source/CPack/cmCPackNSISGenerator.cxx index e46b2cf..f75a750 100644 --- a/Source/CPack/cmCPackNSISGenerator.cxx +++ b/Source/CPack/cmCPackNSISGenerator.cxx @@ -305,7 +305,7 @@ int cmCPackNSISGenerator::PackageFiles() nsisCmd.c_str(), &output, &output, &retVal, nullptr, this->GeneratorVerbose, cmDuration::zero()); if (!res || retVal) { - cmGeneratedFileStream ofs(tmpFile.c_str()); + cmGeneratedFileStream ofs(tmpFile); ofs << "# Run command: " << nsisCmd << std::endl << "# Output:" << std::endl << output << std::endl; @@ -416,7 +416,7 @@ int cmCPackNSISGenerator::InitializeInternal() const char* topDir = this->GetOption("CPACK_TOPLEVEL_DIRECTORY"); std::string tmpFile = topDir ? topDir : "."; tmpFile += "/NSISOutput.log"; - cmGeneratedFileStream ofs(tmpFile.c_str()); + cmGeneratedFileStream ofs(tmpFile); ofs << "# Run command: " << nsisCmd << std::endl << "# Output:" << std::endl << output << std::endl; @@ -726,7 +726,7 @@ std::string cmCPackNSISGenerator::CreateComponentDescription( cmSystemTools::IsOn(this->GetOption("CPACK_ZIP_NEED_QUOTES")); unsigned long totalSize = 0; { // the scope is needed for cmGeneratedFileStream - cmGeneratedFileStream out(zipListFileName.c_str()); + cmGeneratedFileStream out(zipListFileName); for (std::string const& file : component->Files) { if (needQuotesInFile) { out << "\""; @@ -754,7 +754,7 @@ std::string cmCPackNSISGenerator::CreateComponentDescription( if (!res || retVal) { std::string tmpFile = this->GetOption("CPACK_TOPLEVEL_DIRECTORY"); tmpFile += "/CompressZip.log"; - cmGeneratedFileStream ofs(tmpFile.c_str()); + cmGeneratedFileStream ofs(tmpFile); ofs << "# Run command: " << cmd << std::endl << "# Output:" << std::endl << output << std::endl; diff --git a/Source/CTest/cmCTestBuildHandler.cxx b/Source/CTest/cmCTestBuildHandler.cxx index 8527d54..3f6654b 100644 --- a/Source/CTest/cmCTestBuildHandler.cxx +++ b/Source/CTest/cmCTestBuildHandler.cxx @@ -589,7 +589,7 @@ void cmCTestBuildHandler::GenerateXMLLogScraped(cmXMLWriter& xml) for (cmCTestCompileErrorWarningRex& rit : this->ErrorWarningFileLineRegex) { cmsys::RegularExpression* re = &rit.RegularExpression; - if (re->find(cm->Text.c_str())) { + if (re->find(cm->Text)) { cm->SourceFile = re->match(rit.FileIndex); // At this point we need to make this->SourceFile relative to // the source root of the project, so cvs links will work @@ -742,7 +742,7 @@ void cmCTestBuildHandler::LaunchHelper::WriteLauncherConfig() // Give some testing configuration information to the launcher. std::string fname = this->Handler->CTestLaunchDir; fname += "/CTestLaunchConfig.cmake"; - cmGeneratedFileStream fout(fname.c_str()); + cmGeneratedFileStream fout(fname); std::string srcdir = this->CTest->GetCTestConfiguration("SourceDirectory"); fout << "set(CTEST_SOURCE_DIRECTORY \"" << srcdir << "\")\n"; } @@ -757,7 +757,7 @@ void cmCTestBuildHandler::LaunchHelper::WriteScrapeMatchers( fname += "/Custom"; fname += purpose; fname += ".txt"; - cmGeneratedFileStream fout(fname.c_str()); + cmGeneratedFileStream fout(fname); for (std::string const& m : matchers) { fout << m << "\n"; } diff --git a/Source/CTest/cmCTestCoverageHandler.cxx b/Source/CTest/cmCTestCoverageHandler.cxx index 43a2bae..1d9a5f7 100644 --- a/Source/CTest/cmCTestCoverageHandler.cxx +++ b/Source/CTest/cmCTestCoverageHandler.cxx @@ -1075,7 +1075,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( if (line.empty()) { // Ignore empty line; probably style 2 - } else if (st1re1.find(line.c_str())) { + } else if (st1re1.find(line)) { if (gcovStyle == 0) { gcovStyle = 1; } @@ -1088,7 +1088,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( actualSourceFile.clear(); sourceFile = st1re1.match(2); - } else if (st1re2.find(line.c_str())) { + } else if (st1re2.find(line)) { if (gcovStyle == 0) { gcovStyle = 1; } @@ -1100,7 +1100,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( } gcovFile = st1re2.match(1); - } else if (st2re1.find(line.c_str())) { + } else if (st2re1.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } @@ -1113,7 +1113,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( actualSourceFile.clear(); sourceFile = st2re1.match(1); - } else if (st2re2.find(line.c_str())) { + } else if (st2re2.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } @@ -1123,7 +1123,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( cont->Error++; break; } - } else if (st2re3.find(line.c_str())) { + } else if (st2re3.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } @@ -1135,7 +1135,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( } gcovFile = st2re3.match(2); - } else if (st2re4.find(line.c_str())) { + } else if (st2re4.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } @@ -1150,7 +1150,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( "Warning: " << st2re4.match(1) << " had unexpected EOF" << std::endl, this->Quiet); - } else if (st2re5.find(line.c_str())) { + } else if (st2re5.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } @@ -1165,7 +1165,7 @@ int cmCTestCoverageHandler::HandleGCovCoverage( "Warning: Cannot open file: " << st2re5.match(1) << std::endl, this->Quiet); - } else if (st2re6.find(line.c_str())) { + } else if (st2re6.find(line)) { if (gcovStyle == 0) { gcovStyle = 2; } diff --git a/Source/CTest/cmCTestLaunch.cxx b/Source/CTest/cmCTestLaunch.cxx index 0bffde3..6cd1c09 100644 --- a/Source/CTest/cmCTestLaunch.cxx +++ b/Source/CTest/cmCTestLaunch.cxx @@ -345,7 +345,7 @@ void cmCTestLaunch::WriteXML() logXML += ".xml"; // Use cmGeneratedFileStream to atomically create the report file. - cmGeneratedFileStream fxml(logXML.c_str()); + cmGeneratedFileStream fxml(logXML); cmXMLWriter xml(fxml, 2); cmXMLElement e2(xml, "Failure"); e2.Attribute("type", this->IsError() ? "Error" : "Warning"); @@ -584,7 +584,7 @@ bool cmCTestLaunch::Match(std::string const& line, std::vector<cmsys::RegularExpression>& regexps) { for (cmsys::RegularExpression& r : regexps) { - if (r.find(line.c_str())) { + if (r.find(line)) { return true; } } diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 927797a..ef0a49d 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -49,7 +49,7 @@ void cmCTestRunTest::CheckOutput(std::string const& line) // Check for TIMEOUT_AFTER_MATCH property. if (!this->TestProperties->TimeoutRegularExpressions.empty()) { for (auto& reg : this->TestProperties->TimeoutRegularExpressions) { - if (reg.first.find(this->ProcessOutput.c_str())) { + if (reg.first.find(this->ProcessOutput)) { cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->GetIndex() << ": " @@ -148,7 +148,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) this->FailedDependencies.empty()) { bool found = false; for (auto& pass : this->TestProperties->RequiredRegularExpressions) { - if (pass.first.find(this->ProcessOutput.c_str())) { + if (pass.first.find(this->ProcessOutput)) { found = true; reason = "Required regular expression found."; break; @@ -168,7 +168,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) if (!this->TestProperties->ErrorRegularExpressions.empty() && this->FailedDependencies.empty()) { for (auto& pass : this->TestProperties->ErrorRegularExpressions) { - if (pass.first.find(this->ProcessOutput.c_str())) { + if (pass.first.find(this->ProcessOutput)) { reason = "Error regular expression found in output."; reason += " Regex=["; reason += pass.second; @@ -616,10 +616,10 @@ void cmCTestRunTest::DartProcessing() { if (!this->ProcessOutput.empty() && this->ProcessOutput.find("<DartMeasurement") != std::string::npos) { - if (this->TestHandler->DartStuff.find(this->ProcessOutput.c_str())) { + if (this->TestHandler->DartStuff.find(this->ProcessOutput)) { this->TestResult.DartString = this->TestHandler->DartStuff.match(1); // keep searching and replacing until none are left - while (this->TestHandler->DartStuff1.find(this->ProcessOutput.c_str())) { + while (this->TestHandler->DartStuff1.find(this->ProcessOutput)) { // replace the exact match for the string cmSystemTools::ReplaceString( this->ProcessOutput, this->TestHandler->DartStuff1.match(1).c_str(), diff --git a/Source/CTest/cmCTestScriptHandler.cxx b/Source/CTest/cmCTestScriptHandler.cxx index 333c899..6b62bb4 100644 --- a/Source/CTest/cmCTestScriptHandler.cxx +++ b/Source/CTest/cmCTestScriptHandler.cxx @@ -792,7 +792,7 @@ int cmCTestScriptHandler::RunConfigurationDashboard() cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Write CMake output to file: " << cmakeOutputFile << std::endl); - cmGeneratedFileStream fout(cmakeOutputFile.c_str()); + cmGeneratedFileStream fout(cmakeOutputFile); if (fout) { fout << output.c_str(); } else { @@ -856,7 +856,7 @@ bool cmCTestScriptHandler::WriteInitialCache(const char* directory, { std::string cacheFile = directory; cacheFile += "/CMakeCache.txt"; - cmGeneratedFileStream fout(cacheFile.c_str()); + cmGeneratedFileStream fout(cacheFile); if (!fout) { return false; } diff --git a/Source/CTest/cmCTestStartCommand.cxx b/Source/CTest/cmCTestStartCommand.cxx index 3911540..47006c1 100644 --- a/Source/CTest/cmCTestStartCommand.cxx +++ b/Source/CTest/cmCTestStartCommand.cxx @@ -121,7 +121,7 @@ bool cmCTestStartCommand::InitialPass(std::vector<std::string> const& args, // Log startup actions. std::string startLogFile = binaryDir + "/Testing/Temporary/LastStart.log"; - cmGeneratedFileStream ofs(startLogFile.c_str()); + cmGeneratedFileStream ofs(startLogFile); if (!ofs) { cmCTestLog(this->CTest, ERROR_MESSAGE, "Cannot create log file: LastStart.log" << std::endl); diff --git a/Source/CTest/cmCTestSubmitHandler.cxx b/Source/CTest/cmCTestSubmitHandler.cxx index 4c2b75e..cbed40e 100644 --- a/Source/CTest/cmCTestSubmitHandler.cxx +++ b/Source/CTest/cmCTestSubmitHandler.cxx @@ -454,6 +454,10 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP(const std::string& localprefix, // specify target ::curl_easy_setopt(curl, CURLOPT_URL, upload_as.c_str()); + // CURLAUTH_BASIC is default, and here we allow additional methods, + // including more secure ones + ::curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY); + // now specify which file to upload ::curl_easy_setopt(curl, CURLOPT_INFILE, ftpfile); diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 91b92a3..d1d8d08 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -2310,7 +2310,7 @@ bool cmCTestTestHandler::AddTest(const std::vector<std::string>& args) this->Quiet); if (this->UseExcludeRegExpFlag && this->UseExcludeRegExpFirst && - this->ExcludeTestsRegularExpression.find(testname.c_str())) { + this->ExcludeTestsRegularExpression.find(testname)) { return true; } if (this->MemCheck) { @@ -2365,10 +2365,10 @@ bool cmCTestTestHandler::AddTest(const std::vector<std::string>& args) test.SkipReturnCode = -1; test.PreviousRuns = 0; if (this->UseIncludeRegExpFlag && - !this->IncludeTestsRegularExpression.find(testname.c_str())) { + !this->IncludeTestsRegularExpression.find(testname)) { test.IsInBasedOnREOptions = false; } else if (this->UseExcludeRegExpFlag && !this->UseExcludeRegExpFirst && - this->ExcludeTestsRegularExpression.find(testname.c_str())) { + this->ExcludeTestsRegularExpression.find(testname)) { test.IsInBasedOnREOptions = false; } this->TestList.push_back(test); diff --git a/Source/cmBuildNameCommand.cxx b/Source/cmBuildNameCommand.cxx index 5f54338..ddff686 100644 --- a/Source/cmBuildNameCommand.cxx +++ b/Source/cmBuildNameCommand.cxx @@ -40,8 +40,8 @@ bool cmBuildNameCommand::InitialPass(std::vector<std::string> const& args, cmSystemTools::RunSingleCommand("uname -a", &buildname, &buildname); if (!buildname.empty()) { std::string RegExp = "([^ ]*) [^ ]* ([^ ]*) "; - cmsys::RegularExpression reg(RegExp.c_str()); - if (reg.find(buildname.c_str())) { + cmsys::RegularExpression reg(RegExp); + if (reg.find(buildname)) { buildname = reg.match(1) + "-" + reg.match(2); } } diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 95edd8b..3d3d526 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -818,7 +818,7 @@ bool cmCTest::OpenOutputFile(const std::string& path, const std::string& name, } } std::string filename = testingDir + "/" + name; - stream.Open(filename.c_str()); + stream.Open(filename); if (!stream) { cmCTestLog(this, ERROR_MESSAGE, "Problem opening file: " << filename << std::endl); diff --git a/Source/cmCacheManager.cxx b/Source/cmCacheManager.cxx index c6a1cff..5091c97 100644 --- a/Source/cmCacheManager.cxx +++ b/Source/cmCacheManager.cxx @@ -234,7 +234,7 @@ bool cmCacheManager::SaveCache(const std::string& path, cmMessenger* messenger) { std::string cacheFile = path; cacheFile += "/CMakeCache.txt"; - cmGeneratedFileStream fout(cacheFile.c_str()); + cmGeneratedFileStream fout(cacheFile); fout.SetCopyIfDifferent(true); if (!fout) { cmSystemTools::Error("Unable to open cache file for save. ", diff --git a/Source/cmDependsC.cxx b/Source/cmDependsC.cxx index 34c0e94..f7dd33b 100644 --- a/Source/cmDependsC.cxx +++ b/Source/cmDependsC.cxx @@ -174,7 +174,7 @@ bool cmDependsC::WriteDependencies(const std::set<std::string>& sources, // Complain if the file cannot be found and matches the complain // regex. if (fullName.empty() && - this->IncludeRegexComplain.find(current.FileName.c_str())) { + this->IncludeRegexComplain.find(current.FileName)) { cmSystemTools::Error("Cannot find file \"", current.FileName.c_str(), "\"."); return false; @@ -359,7 +359,7 @@ void cmDependsC::Scan(std::istream& is, const char* directory, } // Match include directives. - if (this->IncludeRegexLine.find(line.c_str())) { + if (this->IncludeRegexLine.find(line)) { // Get the file being included. UnscannedEntry entry; entry.FileName = this->IncludeRegexLine.match(2); @@ -383,7 +383,7 @@ void cmDependsC::Scan(std::istream& is, const char* directory, // file their own directory by simply using "filename.h" (#12619) // This kind of problem will be fixed when a more // preprocessor-like implementation of this scanner is created. - if (this->IncludeRegexScan.find(entry.FileName.c_str())) { + if (this->IncludeRegexScan.find(entry.FileName)) { newCacheEntry->UnscannedEntries.push_back(entry); if (this->Encountered.find(entry.FileName) == this->Encountered.end()) { @@ -449,7 +449,7 @@ void cmDependsC::ParseTransform(std::string const& xform) void cmDependsC::TransformLine(std::string& line) { // Check for a transform rule match. Return if none. - if (!this->IncludeRegexTransform.find(line.c_str())) { + if (!this->IncludeRegexTransform.find(line)) { return; } TransformRulesType::const_iterator tri = diff --git a/Source/cmDependsFortran.cxx b/Source/cmDependsFortran.cxx index dbea15a..a04cee7 100644 --- a/Source/cmDependsFortran.cxx +++ b/Source/cmDependsFortran.cxx @@ -178,7 +178,7 @@ bool cmDependsFortran::Finalize(std::ostream& makeDepends, // Store the list of modules provided by this target. std::string fiName = this->TargetDirectory; fiName += "/fortran.internal"; - cmGeneratedFileStream fiStream(fiName.c_str()); + cmGeneratedFileStream fiStream(fiName); fiStream << "# The fortran modules provided by this target.\n"; fiStream << "provides\n"; std::set<std::string> const& provides = this->Internal->TargetProvides; @@ -190,7 +190,7 @@ bool cmDependsFortran::Finalize(std::ostream& makeDepends, if (!provides.empty()) { std::string fcName = this->TargetDirectory; fcName += "/cmake_clean_Fortran.cmake"; - cmGeneratedFileStream fcStream(fcName.c_str()); + cmGeneratedFileStream fcStream(fcName); fcStream << "# Remove fortran modules provided by this target.\n"; fcStream << "FILE(REMOVE"; std::string currentBinDir = diff --git a/Source/cmExportCommand.cxx b/Source/cmExportCommand.cxx index 4e1d771..c6ec5bf 100644 --- a/Source/cmExportCommand.cxx +++ b/Source/cmExportCommand.cxx @@ -237,7 +237,7 @@ bool cmExportCommand::HandlePackage(std::vector<std::string> const& args) } const char* packageExpr = "^[A-Za-z0-9_.-]+$"; cmsys::RegularExpression packageRegex(packageExpr); - if (!packageRegex.find(package.c_str())) { + if (!packageRegex.find(package)) { std::ostringstream e; e << "PACKAGE given invalid package name \"" << package << "\". " << "Package names must match \"" << packageExpr << "\"."; @@ -339,7 +339,7 @@ void cmExportCommand::StorePackageRegistryDir(std::string const& package, fname += "/"; fname += hash; if (!cmSystemTools::FileExists(fname)) { - cmGeneratedFileStream entry(fname.c_str(), true); + cmGeneratedFileStream entry(fname, true); if (entry) { entry << content << "\n"; } else { diff --git a/Source/cmExportFileGenerator.cxx b/Source/cmExportFileGenerator.cxx index ec68fce..75d3374 100644 --- a/Source/cmExportFileGenerator.cxx +++ b/Source/cmExportFileGenerator.cxx @@ -75,7 +75,7 @@ bool cmExportFileGenerator::GenerateImportFile() } else { // Generate atomically and with copy-if-different. std::unique_ptr<cmGeneratedFileStream> ap( - new cmGeneratedFileStream(this->MainImportFile.c_str(), true)); + new cmGeneratedFileStream(this->MainImportFile, true)); ap->SetCopyIfDifferent(true); foutPtr = std::move(ap); } @@ -1174,6 +1174,11 @@ bool cmExportFileGenerator::PopulateExportProperties( return false; } auto propertyValue = targetProperties.GetPropertyValue(prop); + if (propertyValue == nullptr) { + // Asked to export a property that isn't defined on the target. Do not + // consider this an error, there's just nothing to export. + continue; + } std::string evaluatedValue = cmGeneratorExpression::Preprocess( propertyValue, cmGeneratorExpression::StripAllGeneratorExpressions); if (evaluatedValue != propertyValue) { diff --git a/Source/cmExportInstallFileGenerator.cxx b/Source/cmExportInstallFileGenerator.cxx index 3595708..bfb7a05 100644 --- a/Source/cmExportInstallFileGenerator.cxx +++ b/Source/cmExportInstallFileGenerator.cxx @@ -288,7 +288,7 @@ bool cmExportInstallFileGenerator::GenerateImportFileConfig( fileName += this->FileExt; // Open the output file to generate it. - cmGeneratedFileStream exportFileStream(fileName.c_str(), true); + cmGeneratedFileStream exportFileStream(fileName, true); if (!exportFileStream) { std::string se = cmSystemTools::GetLastSystemError(); std::ostringstream e; diff --git a/Source/cmExportLibraryDependenciesCommand.cxx b/Source/cmExportLibraryDependenciesCommand.cxx index 50f8cb0..b4b2962 100644 --- a/Source/cmExportLibraryDependenciesCommand.cxx +++ b/Source/cmExportLibraryDependenciesCommand.cxx @@ -54,7 +54,7 @@ void cmExportLibraryDependenciesCommand::ConstFinalPass() const cm::make_unique<cmsys::ofstream>(this->Filename.c_str(), std::ios::app); } else { std::unique_ptr<cmGeneratedFileStream> ap( - new cmGeneratedFileStream(this->Filename.c_str(), true)); + new cmGeneratedFileStream(this->Filename, true)); ap->SetCopyIfDifferent(true); foutPtr = std::move(ap); } diff --git a/Source/cmExtraCodeBlocksGenerator.cxx b/Source/cmExtraCodeBlocksGenerator.cxx index b636c73..fbf6560 100644 --- a/Source/cmExtraCodeBlocksGenerator.cxx +++ b/Source/cmExtraCodeBlocksGenerator.cxx @@ -199,7 +199,7 @@ void cmExtraCodeBlocksGenerator::CreateNewProjectFile( const std::vector<cmLocalGenerator*>& lgs, const std::string& filename) { const cmMakefile* mf = lgs[0]->GetMakefile(); - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } @@ -476,7 +476,7 @@ std::string cmExtraCodeBlocksGenerator::CreateDummyTargetFile( filename += "/"; filename += target->GetName(); filename += ".objlib"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (fout) { /* clang-format off */ fout << "# This is a dummy file for the OBJECT library " diff --git a/Source/cmExtraCodeLiteGenerator.cxx b/Source/cmExtraCodeLiteGenerator.cxx index b76efb2..582a2e7 100644 --- a/Source/cmExtraCodeLiteGenerator.cxx +++ b/Source/cmExtraCodeLiteGenerator.cxx @@ -77,7 +77,7 @@ void cmExtraCodeLiteGenerator::Generate() } } - cmGeneratedFileStream fout(workspaceFileName.c_str()); + cmGeneratedFileStream fout(workspaceFileName); cmXMLWriter xml(fout); xml.StartDocument("utf-8"); @@ -249,7 +249,7 @@ void cmExtraCodeLiteGenerator::CreateNewProjectFile( const std::vector<cmLocalGenerator*>& lgs, const std::string& filename) { const cmMakefile* mf = lgs[0]->GetMakefile(); - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } @@ -547,7 +547,7 @@ void cmExtraCodeLiteGenerator::CreateNewProjectFile( const cmGeneratorTarget* gt, const std::string& filename) { const cmMakefile* mf = gt->Makefile; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } diff --git a/Source/cmExtraEclipseCDT4Generator.cxx b/Source/cmExtraEclipseCDT4Generator.cxx index 14448f5..313d46b 100644 --- a/Source/cmExtraEclipseCDT4Generator.cxx +++ b/Source/cmExtraEclipseCDT4Generator.cxx @@ -98,7 +98,7 @@ void cmExtraEclipseCDT4Generator::Generate() std::string eclipseVersion = mf->GetSafeDefinition("CMAKE_ECLIPSE_VERSION"); cmsys::RegularExpression regex(".*([0-9]+\\.[0-9]+).*"); - if (regex.find(eclipseVersion.c_str())) { + if (regex.find(eclipseVersion)) { unsigned int majorVersion = 0; unsigned int minorVersion = 0; int res = @@ -176,7 +176,7 @@ void cmExtraEclipseCDT4Generator::CreateSourceProjectFile() this->GetPathBasename(this->HomeDirectory)); const std::string filename = this->HomeDirectory + "/.project"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } @@ -261,7 +261,7 @@ void cmExtraEclipseCDT4Generator::CreateProjectFile() const std::string filename = this->HomeOutputDirectory + "/.project"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } @@ -555,7 +555,7 @@ void cmExtraEclipseCDT4Generator::AppendIncludeDirectories( // Frameworks/ part has to be stripped // /System/Library/Frameworks/GLUT.framework/Headers cmsys::RegularExpression frameworkRx("(.+/Frameworks)/.+\\.framework/"); - if (frameworkRx.find(dir.c_str())) { + if (frameworkRx.find(dir)) { dir = frameworkRx.match(1); } @@ -582,7 +582,7 @@ void cmExtraEclipseCDT4Generator::CreateCProjectFile() const const std::string filename = this->HomeOutputDirectory + "/.cproject"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } diff --git a/Source/cmExtraKateGenerator.cxx b/Source/cmExtraKateGenerator.cxx index df18715..f5c4c93 100644 --- a/Source/cmExtraKateGenerator.cxx +++ b/Source/cmExtraKateGenerator.cxx @@ -58,7 +58,7 @@ void cmExtraKateGenerator::CreateKateProjectFile( { std::string filename = lg->GetBinaryDirectory(); filename += "/.kateproject"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } @@ -215,7 +215,7 @@ void cmExtraKateGenerator::CreateDummyKateProjectFile( filename += "/"; filename += this->ProjectName; filename += ".kateproject"; - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } diff --git a/Source/cmExtraSublimeTextGenerator.cxx b/Source/cmExtraSublimeTextGenerator.cxx index fb85a68..14e7927 100644 --- a/Source/cmExtraSublimeTextGenerator.cxx +++ b/Source/cmExtraSublimeTextGenerator.cxx @@ -91,7 +91,7 @@ void cmExtraSublimeTextGenerator::CreateNewProjectFile( { const cmMakefile* mf = lgs[0]->GetMakefile(); - cmGeneratedFileStream fout(filename.c_str()); + cmGeneratedFileStream fout(filename); if (!fout) { return; } diff --git a/Source/cmFileCommand.cxx b/Source/cmFileCommand.cxx index 402ceb2..0c80319 100644 --- a/Source/cmFileCommand.cxx +++ b/Source/cmFileCommand.cxx @@ -663,7 +663,7 @@ bool cmFileCommand::HandleStringsCommand(std::vector<std::string> const& args) // The current line has been terminated. Check if the current // string matches the requirements. The length may now be as // low as zero since blank lines are allowed. - if (s.length() >= minlen && (!have_regex || regex.find(s.c_str()))) { + if (s.length() >= minlen && (!have_regex || regex.find(s))) { output_size += static_cast<int>(s.size()) + 1; if (limit_output >= 0 && output_size >= limit_output) { s.clear(); @@ -679,7 +679,7 @@ bool cmFileCommand::HandleStringsCommand(std::vector<std::string> const& args) // string matches the requirements. We require that the length // be at least one no matter what the user specified. if (s.length() >= minlen && !s.empty() && - (!have_regex || regex.find(s.c_str()))) { + (!have_regex || regex.find(s))) { output_size += static_cast<int>(s.size()) + 1; if (limit_output >= 0 && output_size >= limit_output) { s.clear(); @@ -696,7 +696,7 @@ bool cmFileCommand::HandleStringsCommand(std::vector<std::string> const& args) if (maxlen > 0 && s.size() == maxlen) { // Terminate a string if the maximum length is reached. - if (s.length() >= minlen && (!have_regex || regex.find(s.c_str()))) { + if (s.length() >= minlen && (!have_regex || regex.find(s))) { output_size += static_cast<int>(s.size()) + 1; if (limit_output >= 0 && output_size >= limit_output) { s.clear(); @@ -712,7 +712,7 @@ bool cmFileCommand::HandleStringsCommand(std::vector<std::string> const& args) // input file or the input size limit. Check if the current string // matches the requirements. if ((!limit_count || strings.size() < limit_count) && !s.empty() && - s.length() >= minlen && (!have_regex || regex.find(s.c_str()))) { + s.length() >= minlen && (!have_regex || regex.find(s))) { output_size += static_cast<int>(s.size()) + 1; if (limit_output < 0 || output_size < limit_output) { strings.push_back(s); diff --git a/Source/cmFindPackageCommand.cxx b/Source/cmFindPackageCommand.cxx index 7ac0cdf..96de6ad 100644 --- a/Source/cmFindPackageCommand.cxx +++ b/Source/cmFindPackageCommand.cxx @@ -337,7 +337,7 @@ bool cmFindPackageCommand::InitialPass(std::vector<std::string> const& args, return false; } this->Configs.push_back(args[i]); - } else if (!haveVersion && version.find(args[i].c_str())) { + } else if (!haveVersion && version.find(args[i])) { haveVersion = true; this->Version = args[i]; } else { @@ -487,7 +487,7 @@ bool cmFindPackageCommand::InitialPass(std::vector<std::string> const& args, this->SetModuleVariables(components); - // See if there is a Find<package>.cmake module. + // See if there is a Find<PackageName>.cmake module. if (this->UseFindModules) { bool foundModule = false; if (!this->FindModule(foundModule)) { @@ -538,7 +538,7 @@ bool cmFindPackageCommand::InitialPass(std::vector<std::string> const& args, } // No find module. Assume the project has a CMake config file. Use - // a <package>_DIR cache variable to locate it. + // a <PackageName>_DIR cache variable to locate it. this->Variable = this->Name; this->Variable += "_DIR"; diff --git a/Source/cmFindPackageCommand.h b/Source/cmFindPackageCommand.h index d8c7ab3..48f17ef 100644 --- a/Source/cmFindPackageCommand.h +++ b/Source/cmFindPackageCommand.h @@ -7,6 +7,7 @@ #include "cm_kwiml.h" #include <cstddef> +#include <functional> #include <map> #include <set> #include <string> @@ -15,14 +16,12 @@ // IWYU insists we should forward-declare instead of including <functional>, // but we cannot forward-declare reliably because some C++ standard libraries // put the template in an inline namespace. -#ifdef CMAKE_IWYU +#ifdef CMAKE_IWYU_FORWARD_STD_HASH /* clang-format off */ namespace std { template <class T> struct hash; } /* clang-format on */ -#else -# include <functional> #endif #include "cmFindCommon.h" diff --git a/Source/cmGeneratedFileStream.cxx b/Source/cmGeneratedFileStream.cxx index 6bdf3a4..186ec8c 100644 --- a/Source/cmGeneratedFileStream.cxx +++ b/Source/cmGeneratedFileStream.cxx @@ -24,8 +24,8 @@ cmGeneratedFileStream::cmGeneratedFileStream(Encoding encoding) #endif } -cmGeneratedFileStream::cmGeneratedFileStream(const char* name, bool quiet, - Encoding encoding) +cmGeneratedFileStream::cmGeneratedFileStream(std::string const& name, + bool quiet, Encoding encoding) : cmGeneratedFileStreamBase(name) , Stream(TempName.c_str()) { @@ -54,7 +54,7 @@ cmGeneratedFileStream::~cmGeneratedFileStream() this->Okay = !this->fail(); } -cmGeneratedFileStream& cmGeneratedFileStream::Open(const char* name, +cmGeneratedFileStream& cmGeneratedFileStream::Open(std::string const& name, bool quiet, bool binaryFlag) { // Store the file name and construct the temporary file name. @@ -114,7 +114,7 @@ cmGeneratedFileStreamBase::cmGeneratedFileStreamBase() { } -cmGeneratedFileStreamBase::cmGeneratedFileStreamBase(const char* name) +cmGeneratedFileStreamBase::cmGeneratedFileStreamBase(std::string const& name) : Name() , TempName() , CopyIfDifferent(false) @@ -130,7 +130,7 @@ cmGeneratedFileStreamBase::~cmGeneratedFileStreamBase() this->Close(); } -void cmGeneratedFileStreamBase::Open(const char* name) +void cmGeneratedFileStreamBase::Open(std::string const& name) { // Save the original name of the file. this->Name = name; @@ -168,12 +168,12 @@ bool cmGeneratedFileStreamBase::Close() // destination atomically. if (this->Compress) { std::string gzname = this->TempName + ".temp.gz"; - if (this->CompressFile(this->TempName.c_str(), gzname.c_str())) { - this->RenameFile(gzname.c_str(), resname.c_str()); + if (this->CompressFile(this->TempName, gzname)) { + this->RenameFile(gzname, resname); } cmSystemTools::RemoveFile(gzname); } else { - this->RenameFile(this->TempName.c_str(), resname.c_str()); + this->RenameFile(this->TempName, resname); } replaced = true; @@ -188,10 +188,10 @@ bool cmGeneratedFileStreamBase::Close() } #ifdef CMAKE_BUILD_WITH_CMAKE -int cmGeneratedFileStreamBase::CompressFile(const char* oldname, - const char* newname) +int cmGeneratedFileStreamBase::CompressFile(std::string const& oldname, + std::string const& newname) { - gzFile gf = gzopen(newname, "w"); + gzFile gf = gzopen(newname.c_str(), "w"); if (!gf) { return 0; } @@ -214,16 +214,17 @@ int cmGeneratedFileStreamBase::CompressFile(const char* oldname, return 1; } #else -int cmGeneratedFileStreamBase::CompressFile(const char*, const char*) +int cmGeneratedFileStreamBase::CompressFile(std::string const&, + std::string const&) { return 0; } #endif -int cmGeneratedFileStreamBase::RenameFile(const char* oldname, - const char* newname) +int cmGeneratedFileStreamBase::RenameFile(std::string const& oldname, + std::string const& newname) { - return cmSystemTools::RenameFile(oldname, newname); + return cmSystemTools::RenameFile(oldname.c_str(), newname.c_str()); } void cmGeneratedFileStream::SetName(const std::string& fname) diff --git a/Source/cmGeneratedFileStream.h b/Source/cmGeneratedFileStream.h index 48f93c5..dacd166 100644 --- a/Source/cmGeneratedFileStream.h +++ b/Source/cmGeneratedFileStream.h @@ -20,7 +20,7 @@ protected: cmGeneratedFileStreamBase(); // This constructor prepares the temporary output file. - cmGeneratedFileStreamBase(const char* name); + cmGeneratedFileStreamBase(std::string const& name); // The destructor renames the temporary output file to the real name. ~cmGeneratedFileStreamBase(); @@ -29,14 +29,14 @@ protected: // called before the real stream is opened. Close is always called // after the real stream is closed and Okay is set to whether the // real stream was still valid for writing when it was closed. - void Open(const char* name); + void Open(std::string const& name); bool Close(); // Internal file replacement implementation. - int RenameFile(const char* oldname, const char* newname); + int RenameFile(std::string const& oldname, std::string const& newname); // Internal file compression implementation. - int CompressFile(const char* oldname, const char* newname); + int CompressFile(std::string const& oldname, std::string const& newname); // The name of the final destination file for the output. std::string Name; @@ -87,7 +87,7 @@ public: * file cannot be opened an error message is produced unless the * second argument is set to true. */ - cmGeneratedFileStream(const char* name, bool quiet = false, + cmGeneratedFileStream(std::string const& name, bool quiet = false, Encoding encoding = codecvt::None); /** @@ -103,7 +103,7 @@ public: * temporary file. If the file cannot be opened an error message is * produced unless the second argument is set to true. */ - cmGeneratedFileStream& Open(const char* name, bool quiet = false, + cmGeneratedFileStream& Open(std::string const& name, bool quiet = false, bool binaryFlag = false); /** diff --git a/Source/cmGeneratorExpression.cxx b/Source/cmGeneratorExpression.cxx index 64ec30d..b1b2b88 100644 --- a/Source/cmGeneratorExpression.cxx +++ b/Source/cmGeneratorExpression.cxx @@ -38,7 +38,7 @@ cmGeneratorExpression::~cmGeneratorExpression() { } -const char* cmCompiledGeneratorExpression::Evaluate( +const std::string& cmCompiledGeneratorExpression::Evaluate( cmLocalGenerator* lg, const std::string& config, bool quiet, const cmGeneratorTarget* headTarget, cmGeneratorExpressionDAGChecker* dagChecker, @@ -48,7 +48,7 @@ const char* cmCompiledGeneratorExpression::Evaluate( language); } -const char* cmCompiledGeneratorExpression::Evaluate( +const std::string& cmCompiledGeneratorExpression::Evaluate( cmLocalGenerator* lg, const std::string& config, bool quiet, const cmGeneratorTarget* headTarget, const cmGeneratorTarget* currentTarget, cmGeneratorExpressionDAGChecker* dagChecker, @@ -61,12 +61,12 @@ const char* cmCompiledGeneratorExpression::Evaluate( return this->EvaluateWithContext(context, dagChecker); } -const char* cmCompiledGeneratorExpression::EvaluateWithContext( +const std::string& cmCompiledGeneratorExpression::EvaluateWithContext( cmGeneratorExpressionContext& context, cmGeneratorExpressionDAGChecker* dagChecker) const { if (!this->NeedsEvaluation) { - return this->Input.c_str(); + return this->Input; } this->Output.clear(); @@ -97,8 +97,7 @@ const char* cmCompiledGeneratorExpression::EvaluateWithContext( this->DependTargets = context.DependTargets; this->AllTargetsSeen = context.AllTargets; - // TODO: Return a std::string from here instead? - return this->Output.c_str(); + return this->Output; } cmCompiledGeneratorExpression::cmCompiledGeneratorExpression( @@ -391,7 +390,7 @@ const char* cmGeneratorExpressionInterpreter::Evaluate( const char* expression, const std::string& property) { if (this->Target.empty()) { - return this->EvaluateExpression(expression); + return this->EvaluateExpression(expression).c_str(); } // Specify COMPILE_OPTIONS to DAGchecker, same semantic as COMPILE_FLAGS @@ -399,5 +398,5 @@ const char* cmGeneratorExpressionInterpreter::Evaluate( this->Target, property == "COMPILE_FLAGS" ? "COMPILE_OPTIONS" : property, nullptr, nullptr); - return this->EvaluateExpression(expression, &dagChecker); + return this->EvaluateExpression(expression, &dagChecker).c_str(); } diff --git a/Source/cmGeneratorExpression.h b/Source/cmGeneratorExpression.h index 9fd53c6..1ceb7da 100644 --- a/Source/cmGeneratorExpression.h +++ b/Source/cmGeneratorExpression.h @@ -72,16 +72,17 @@ class cmCompiledGeneratorExpression CM_DISABLE_COPY(cmCompiledGeneratorExpression) public: - const char* Evaluate(cmLocalGenerator* lg, const std::string& config, - bool quiet = false, - cmGeneratorTarget const* headTarget = nullptr, - cmGeneratorTarget const* currentTarget = nullptr, - cmGeneratorExpressionDAGChecker* dagChecker = nullptr, - std::string const& language = std::string()) const; - const char* Evaluate(cmLocalGenerator* lg, const std::string& config, - bool quiet, cmGeneratorTarget const* headTarget, - cmGeneratorExpressionDAGChecker* dagChecker, - std::string const& language = std::string()) const; + const std::string& Evaluate( + cmLocalGenerator* lg, const std::string& config, bool quiet = false, + cmGeneratorTarget const* headTarget = nullptr, + cmGeneratorTarget const* currentTarget = nullptr, + cmGeneratorExpressionDAGChecker* dagChecker = nullptr, + std::string const& language = std::string()) const; + const std::string& Evaluate( + cmLocalGenerator* lg, const std::string& config, bool quiet, + cmGeneratorTarget const* headTarget, + cmGeneratorExpressionDAGChecker* dagChecker, + std::string const& language = std::string()) const; /** Get set of targets found during evaluations. */ std::set<cmGeneratorTarget*> const& GetTargets() const @@ -126,7 +127,7 @@ public: std::map<std::string, std::string>& mapping); private: - const char* EvaluateWithContext( + const std::string& EvaluateWithContext( cmGeneratorExpressionContext& context, cmGeneratorExpressionDAGChecker* dagChecker) const; @@ -180,7 +181,7 @@ public: const char* Evaluate(const char* expression) { - return this->EvaluateExpression(expression); + return this->EvaluateExpression(expression).c_str(); } const char* Evaluate(const std::string& expression) { @@ -211,7 +212,7 @@ protected: const std::string& GetTargetName() const { return this->Target; } const std::string& GetLanguage() const { return this->Language; } - const char* EvaluateExpression( + const std::string& EvaluateExpression( const char* expression, cmGeneratorExpressionDAGChecker* dagChecker = nullptr) { diff --git a/Source/cmGeneratorExpressionEvaluationFile.cxx b/Source/cmGeneratorExpressionEvaluationFile.cxx index 99b9261..e7a55e0 100644 --- a/Source/cmGeneratorExpressionEvaluationFile.cxx +++ b/Source/cmGeneratorExpressionEvaluationFile.cxx @@ -55,7 +55,7 @@ void cmGeneratorExpressionEvaluationFile::Generate( std::string outputFileName = this->OutputFileExpr->Evaluate( lg, config, false, nullptr, nullptr, nullptr, lang); - const std::string outputContent = inputExpression->Evaluate( + const std::string& outputContent = inputExpression->Evaluate( lg, config, false, nullptr, nullptr, nullptr, lang); if (cmSystemTools::FileIsFullPath(outputFileName)) { @@ -85,7 +85,7 @@ void cmGeneratorExpressionEvaluationFile::Generate( this->Files.push_back(outputFileName); outputFiles[outputFileName] = outputContent; - cmGeneratedFileStream fout(outputFileName.c_str()); + cmGeneratedFileStream fout(outputFileName); fout.SetCopyIfDifferent(true); fout << outputContent; if (fout.Close() && perm) { diff --git a/Source/cmGeneratorExpressionNode.cxx b/Source/cmGeneratorExpressionNode.cxx index 1594904..3381e6d 100644 --- a/Source/cmGeneratorExpressionNode.cxx +++ b/Source/cmGeneratorExpressionNode.cxx @@ -1126,7 +1126,7 @@ static const struct TargetPropertyNode : public cmGeneratorExpressionNode std::string targetName = parameters.front(); propertyName = parameters[1]; if (!cmGeneratorExpression::IsValidTargetName(targetName)) { - if (!propertyNameValidator.find(propertyName.c_str())) { + if (!propertyNameValidator.find(propertyName)) { ::reportError(context, content->GetOriginalExpression(), "Target name and property name not supported."); return std::string(); diff --git a/Source/cmGeneratorTarget.cxx b/Source/cmGeneratorTarget.cxx index 098c13a..1548374 100644 --- a/Source/cmGeneratorTarget.cxx +++ b/Source/cmGeneratorTarget.cxx @@ -1139,7 +1139,7 @@ void cmGeneratorTarget::ComputeKindedSources(KindedSources& files, std::string cppFileName = xaml + ".cpp"; files.ExpectedXamlHeaders.insert(hFileName); files.ExpectedXamlSources.insert(cppFileName); - } else if (header_regex.find(sf->GetFullPath().c_str())) { + } else if (header_regex.find(sf->GetFullPath())) { kind = SourceKindHeader; } else { kind = SourceKindExtra; @@ -5519,7 +5519,7 @@ void cmGeneratorTarget::ComputeLinkImplementationLibraries( this->GetName(), "LINK_LIBRARIES", nullptr, nullptr); cmGeneratorExpression ge(*btIt); std::unique_ptr<cmCompiledGeneratorExpression> const cge = ge.Parse(*le); - std::string const evaluated = + std::string const& evaluated = cge->Evaluate(this->LocalGenerator, config, false, head, &dagChecker); cmSystemTools::ExpandListArgument(evaluated, llibs); if (cge->GetHadHeadSensitiveCondition()) { diff --git a/Source/cmGlobVerificationManager.cxx b/Source/cmGlobVerificationManager.cxx index e23b6ea..e8959f2 100644 --- a/Source/cmGlobVerificationManager.cxx +++ b/Source/cmGlobVerificationManager.cxx @@ -23,7 +23,7 @@ bool cmGlobVerificationManager::SaveVerificationScript(const std::string& path) cmSystemTools::MakeDirectory(scriptFile); scriptFile += "/VerifyGlobs.cmake"; stampFile += "/cmake.verify_globs"; - cmGeneratedFileStream verifyScriptFile(scriptFile.c_str()); + cmGeneratedFileStream verifyScriptFile(scriptFile); verifyScriptFile.SetCopyIfDifferent(true); if (!verifyScriptFile) { cmSystemTools::Error("Unable to open verification script file for save. ", @@ -170,3 +170,10 @@ void cmGlobVerificationManager::AddCacheEntry( value.Backtraces.emplace_back(variable, backtrace); } } + +void cmGlobVerificationManager::Reset() +{ + this->Cache.clear(); + this->VerifyScript.clear(); + this->VerifyStamp.clear(); +} diff --git a/Source/cmGlobVerificationManager.h b/Source/cmGlobVerificationManager.h index 4508602..cf04c97 100644 --- a/Source/cmGlobVerificationManager.h +++ b/Source/cmGlobVerificationManager.h @@ -37,6 +37,9 @@ protected: const std::string& variable, const cmListFileBacktrace& bt); + ///! Clear the glob cache for state reset. + void Reset(); + ///! Check targets should be written in generated build system. bool DoWriteVerifyTarget() const; diff --git a/Source/cmGlobalGenerator.cxx b/Source/cmGlobalGenerator.cxx index 58821c2..4060269 100644 --- a/Source/cmGlobalGenerator.cxx +++ b/Source/cmGlobalGenerator.cxx @@ -34,6 +34,7 @@ #include "cmMakefile.h" #include "cmOutputConverter.h" #include "cmPolicies.h" +#include "cmQtAutoGen.h" #include "cmQtAutoGenInitializer.h" #include "cmSourceFile.h" #include "cmState.h" @@ -1495,15 +1496,14 @@ bool cmGlobalGenerator::QtAutoGen() continue; } - std::string qtVersionMajor = - cmQtAutoGenInitializer::GetQtMajorVersion(target); + auto qtVersion = cmQtAutoGenInitializer::GetQtVersion(target); // don't do anything if there is no Qt4 or Qt5Core (which contains moc) - if (qtVersionMajor != "4" && qtVersionMajor != "5") { + if (qtVersion.Major != 4 && qtVersion.Major != 5) { continue; } autogenInits.emplace_back(cm::make_unique<cmQtAutoGenInitializer>( - target, mocEnabled, uicEnabled, rccEnabled, qtVersionMajor)); + target, mocEnabled, uicEnabled, rccEnabled, qtVersion)); } } @@ -2904,7 +2904,7 @@ void cmGlobalGenerator::WriteRuleHashes(std::string const& pfile) if (this->RuleHashes.empty()) { cmSystemTools::RemoveFile(pfile); } else { - cmGeneratedFileStream fout(pfile.c_str()); + cmGeneratedFileStream fout(pfile); fout << "# Hashes of file build rules.\n"; for (auto const& rh : this->RuleHashes) { fout.write(rh.second.Data, 32); @@ -2919,7 +2919,7 @@ void cmGlobalGenerator::WriteSummary() std::string fname = this->CMakeInstance->GetHomeOutputDirectory(); fname += cmake::GetCMakeFilesDirectory(); fname += "/TargetDirectories.txt"; - cmGeneratedFileStream fout(fname.c_str()); + cmGeneratedFileStream fout(fname); for (cmLocalGenerator* lg : this->LocalGenerators) { const std::vector<cmGeneratorTarget*>& tgts = lg->GetGeneratorTargets(); @@ -2956,7 +2956,7 @@ void cmGlobalGenerator::WriteSummary(cmGeneratorTarget* target) Json::Value& lj_sources = lj_root["sources"] = Json::arrayValue; cmSystemTools::MakeDirectory(dir); - cmGeneratedFileStream fout(file.c_str()); + cmGeneratedFileStream fout(file); std::vector<std::string> labels; @@ -3030,7 +3030,7 @@ void cmGlobalGenerator::WriteSummary(cmGeneratorTarget* target) } } } - cmGeneratedFileStream json_fout(json_file.c_str()); + cmGeneratedFileStream json_fout(json_file); json_fout << lj_root; } else #endif @@ -3105,7 +3105,7 @@ bool cmGlobalGenerator::GenerateCPackPropertiesFile() return true; } - cmGeneratedFileStream file(path.c_str()); + cmGeneratedFileStream file(path); file << "# CPack properties\n"; for (auto const& i : installedFiles) { diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index 531f79e..677a340 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -763,7 +763,7 @@ void cmGlobalNinjaGenerator::OpenBuildFileStream() // Get a stream where to generate things. if (!this->BuildFileStream) { this->BuildFileStream = new cmGeneratedFileStream( - buildFilePath.c_str(), false, this->GetMakefileEncoding()); + buildFilePath, false, this->GetMakefileEncoding()); if (!this->BuildFileStream) { // An error message is generated by the constructor if it cannot // open the file. @@ -801,7 +801,7 @@ void cmGlobalNinjaGenerator::OpenRulesFileStream() // Get a stream where to generate things. if (!this->RulesFileStream) { this->RulesFileStream = new cmGeneratedFileStream( - rulesFilePath.c_str(), false, this->GetMakefileEncoding()); + rulesFilePath, false, this->GetMakefileEncoding()); if (!this->RulesFileStream) { // An error message is generated by the constructor if it cannot // open the file. @@ -883,8 +883,7 @@ void cmGlobalNinjaGenerator::AddCXXCompileCommand( } // Get a stream where to generate things. - this->CompileCommandsStream = - new cmGeneratedFileStream(buildFilePath.c_str()); + this->CompileCommandsStream = new cmGeneratedFileStream(buildFilePath); *this->CompileCommandsStream << "["; } else { *this->CompileCommandsStream << "," << std::endl; @@ -1706,7 +1705,7 @@ int cmcmd_cmake_ninja_depends(std::vector<std::string>::const_iterator argBeg, } { - cmGeneratedFileStream depfile(arg_dep.c_str()); + cmGeneratedFileStream depfile(arg_dep); depfile << cmSystemTools::ConvertToUnixOutputPath(arg_pp) << ":"; for (std::string const& include : info.Includes) { depfile << " \\\n " << cmSystemTools::ConvertToUnixOutputPath(include); @@ -1729,7 +1728,7 @@ int cmcmd_cmake_ninja_depends(std::vector<std::string>::const_iterator argBeg, } } - cmGeneratedFileStream ddif(arg_ddi.c_str()); + cmGeneratedFileStream ddif(arg_ddi); ddif << ddi; if (!ddif) { cmSystemTools::Error("-E cmake_ninja_depends failed to write ", @@ -1833,7 +1832,7 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile( } } - cmGeneratedFileStream ddf(arg_dd.c_str()); + cmGeneratedFileStream ddf(arg_dd); ddf << "ninja_dyndep_version = 1.0\n"; for (cmFortranObjectInfo const& object : objects) { @@ -1868,7 +1867,7 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile( // use by dependents that reference this target in linked-target-dirs. std::string const target_mods_file = cmSystemTools::GetFilenamePath(arg_dd) + "/FortranModules.json"; - cmGeneratedFileStream tmf(target_mods_file.c_str()); + cmGeneratedFileStream tmf(target_mods_file); tmf << tm; return true; diff --git a/Source/cmGlobalUnixMakefileGenerator3.cxx b/Source/cmGlobalUnixMakefileGenerator3.cxx index 61c42be..db72353 100644 --- a/Source/cmGlobalUnixMakefileGenerator3.cxx +++ b/Source/cmGlobalUnixMakefileGenerator3.cxx @@ -147,7 +147,7 @@ void cmGlobalUnixMakefileGenerator3::Generate() markFileName += "/"; markFileName += cmake::GetCMakeFilesDirectory(); markFileName += "/progress.marks"; - cmGeneratedFileStream markFile(markFileName.c_str()); + cmGeneratedFileStream markFile(markFileName); markFile << this->CountProgressMarksInAll(lg) << "\n"; } @@ -170,8 +170,7 @@ void cmGlobalUnixMakefileGenerator3::AddCXXCompileCommand( std::string commandDatabaseName = std::string(this->GetCMakeInstance()->GetHomeOutputDirectory()) + "/compile_commands.json"; - this->CommandDatabase = - new cmGeneratedFileStream(commandDatabaseName.c_str()); + this->CommandDatabase = new cmGeneratedFileStream(commandDatabaseName); *this->CommandDatabase << "[" << std::endl; } else { *this->CommandDatabase << "," << std::endl; @@ -198,7 +197,7 @@ void cmGlobalUnixMakefileGenerator3::WriteMainMakefile2() this->GetCMakeInstance()->GetHomeOutputDirectory(); makefileName += cmake::GetCMakeFilesDirectory(); makefileName += "/Makefile2"; - cmGeneratedFileStream makefileStream(makefileName.c_str(), false, + cmGeneratedFileStream makefileStream(makefileName, false, this->GetMakefileEncoding()); if (!makefileStream) { return; @@ -270,7 +269,7 @@ void cmGlobalUnixMakefileGenerator3::WriteMainCMakefile() this->GetCMakeInstance()->GetHomeOutputDirectory(); cmakefileName += cmake::GetCMakeFilesDirectory(); cmakefileName += "/Makefile.cmake"; - cmGeneratedFileStream cmakefileStream(cmakefileName.c_str()); + cmGeneratedFileStream cmakefileStream(cmakefileName); if (!cmakefileStream) { return; } @@ -876,7 +875,7 @@ void cmGlobalUnixMakefileGenerator3::RecordTargetProgress( void cmGlobalUnixMakefileGenerator3::TargetProgress::WriteProgressVariables( unsigned long total, unsigned long& current) { - cmGeneratedFileStream fout(this->VariableFile.c_str()); + cmGeneratedFileStream fout(this->VariableFile); for (unsigned long i = 1; i <= this->NumberOfActions; ++i) { fout << "CMAKE_PROGRESS_" << i << " = "; if (total <= 100) { diff --git a/Source/cmGlobalVisualStudio7Generator.cxx b/Source/cmGlobalVisualStudio7Generator.cxx index 21121f2..28cbdc7 100644 --- a/Source/cmGlobalVisualStudio7Generator.cxx +++ b/Source/cmGlobalVisualStudio7Generator.cxx @@ -679,7 +679,7 @@ std::set<std::string> cmGlobalVisualStudio7Generator::IsPartOfDefaultBuild( std::unique_ptr<cmCompiledGeneratorExpression> cge = ge.Parse(propertyValue); if (cmSystemTools::IsOn( - cge->Evaluate(target->GetLocalGenerator(), i))) { + cge->Evaluate(target->GetLocalGenerator(), i).c_str())) { activeConfigs.insert(i); } } diff --git a/Source/cmGraphVizWriter.cxx b/Source/cmGraphVizWriter.cxx index 1bbfbe0..fcdf03f 100644 --- a/Source/cmGraphVizWriter.cxx +++ b/Source/cmGraphVizWriter.cxx @@ -257,7 +257,7 @@ void cmGraphVizWriter::WriteTargetDependersFiles(const char* fileName) currentFilename += ptr.first; currentFilename += ".dependers"; - cmGeneratedFileStream str(currentFilename.c_str()); + cmGeneratedFileStream str(currentFilename); if (!str) { return; } @@ -300,7 +300,7 @@ void cmGraphVizWriter::WritePerTargetFiles(const char* fileName) std::string currentFilename = fileName; currentFilename += "."; currentFilename += ptr.first; - cmGeneratedFileStream str(currentFilename.c_str()); + cmGeneratedFileStream str(currentFilename); if (!str) { return; } diff --git a/Source/cmLocalGenerator.cxx b/Source/cmLocalGenerator.cxx index a669477..5e6ce1a 100644 --- a/Source/cmLocalGenerator.cxx +++ b/Source/cmLocalGenerator.cxx @@ -253,7 +253,7 @@ void cmLocalGenerator::GenerateTestFiles() file += "/"; file += "CTestTestfile.cmake"; - cmGeneratedFileStream fout(file.c_str()); + cmGeneratedFileStream fout(file); fout.SetCopyIfDifferent(true); fout << "# CMake generated Testfile for " << std::endl @@ -429,7 +429,7 @@ void cmLocalGenerator::GenerateInstallRules() toplevel_install = 1; } file += "/cmake_install.cmake"; - cmGeneratedFileStream fout(file.c_str()); + cmGeneratedFileStream fout(file); fout.SetCopyIfDifferent(true); // Write the header. @@ -1971,7 +1971,7 @@ void cmLocalGenerator::AppendCompileOptions( cmsys::RegularExpression r(regex); for (std::string const& opt : options_vec) { - if (r.find(opt.c_str())) { + if (r.find(opt)) { this->AppendFlagEscape(options, opt); } } @@ -2524,12 +2524,12 @@ std::string const& cmLocalGenerator::GetBinaryDirectory() const const char* cmLocalGenerator::GetCurrentBinaryDirectory() const { - return this->StateSnapshot.GetDirectory().GetCurrentBinary(); + return this->StateSnapshot.GetDirectory().GetCurrentBinary().c_str(); } const char* cmLocalGenerator::GetCurrentSourceDirectory() const { - return this->StateSnapshot.GetDirectory().GetCurrentSource(); + return this->StateSnapshot.GetDirectory().GetCurrentSource().c_str(); } std::string cmLocalGenerator::GetTargetDirectory( diff --git a/Source/cmLocalUnixMakefileGenerator3.cxx b/Source/cmLocalUnixMakefileGenerator3.cxx index 9fd2ddb..bc83ce2 100644 --- a/Source/cmLocalUnixMakefileGenerator3.cxx +++ b/Source/cmLocalUnixMakefileGenerator3.cxx @@ -211,8 +211,7 @@ void cmLocalUnixMakefileGenerator3::WriteLocalMakefile() // rules may depend on this file itself. std::string ruleFileNameFull = this->ConvertToFullPath(ruleFileName); cmGeneratedFileStream ruleFileStream( - ruleFileNameFull.c_str(), false, - this->GlobalGenerator->GetMakefileEncoding()); + ruleFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); if (!ruleFileStream) { return; } @@ -433,7 +432,7 @@ void cmLocalUnixMakefileGenerator3::WriteDirectoryInformationFile() infoFileName += "/CMakeDirectoryInformation.cmake"; // Open the output file. - cmGeneratedFileStream infoFileStream(infoFileName.c_str()); + cmGeneratedFileStream infoFileStream(infoFileName); if (!infoFileStream) { return; } @@ -1403,8 +1402,7 @@ bool cmLocalUnixMakefileGenerator3::ScanDependencies( std::string ruleFileNameFull = dir; ruleFileNameFull += "/depend.make"; cmGeneratedFileStream ruleFileStream( - ruleFileNameFull.c_str(), false, - this->GlobalGenerator->GetMakefileEncoding()); + ruleFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); ruleFileStream.SetCopyIfDifferent(true); if (!ruleFileStream) { return false; @@ -1416,7 +1414,7 @@ bool cmLocalUnixMakefileGenerator3::ScanDependencies( std::string internalRuleFileNameFull = dir; internalRuleFileNameFull += "/depend.internal"; cmGeneratedFileStream internalRuleFileStream( - internalRuleFileNameFull.c_str(), false, + internalRuleFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); if (!internalRuleFileStream) { return false; diff --git a/Source/cmLocalVisualStudio7Generator.cxx b/Source/cmLocalVisualStudio7Generator.cxx index c90c5b8..b35b116 100644 --- a/Source/cmLocalVisualStudio7Generator.cxx +++ b/Source/cmLocalVisualStudio7Generator.cxx @@ -712,8 +712,8 @@ void cmLocalVisualStudio7Generator::WriteConfiguration( targetOptions.FixExceptionHandlingDefault(); std::string asmLocation = configName + "/"; targetOptions.AddFlag("AssemblerListingLocation", asmLocation); - targetOptions.Parse(flags.c_str()); - targetOptions.Parse(defineFlags.c_str()); + targetOptions.Parse(flags); + targetOptions.Parse(defineFlags); targetOptions.ParseFinish(); if (!langForClCompile.empty()) { std::vector<std::string> targetDefines; @@ -990,7 +990,7 @@ void cmLocalVisualStudio7Generator::OutputBuildTool( } linkOptions.AddTable(cmLocalVisualStudio7GeneratorLinkFlagTable); - linkOptions.Parse(extraLinkOptions.c_str()); + linkOptions.Parse(extraLinkOptions); cmGeneratorTarget::ModuleDefinitionInfo const* mdi = target->GetModuleDefinitionInfo(configName); if (mdi && !mdi->DefFile.empty()) { @@ -1735,7 +1735,7 @@ bool cmLocalVisualStudio7Generator::WriteGroup( table = cmLocalVisualStudio7GeneratorFortranFlagTable; } Options fileOptions(this, tool, table, gg->ExtraFlagTable); - fileOptions.Parse(fc.CompileFlags.c_str()); + fileOptions.Parse(fc.CompileFlags); fileOptions.AddDefines(fc.CompileDefs); fileOptions.AddDefines(fc.CompileDefsConfig); // validate source level include directories diff --git a/Source/cmMakefile.cxx b/Source/cmMakefile.cxx index c868159..9e14917 100644 --- a/Source/cmMakefile.cxx +++ b/Source/cmMakefile.cxx @@ -1637,12 +1637,12 @@ void cmMakefile::AddSubDirectory(const std::string& srcPath, const char* cmMakefile::GetCurrentSourceDirectory() const { - return this->StateSnapshot.GetDirectory().GetCurrentSource(); + return this->StateSnapshot.GetDirectory().GetCurrentSource().c_str(); } const char* cmMakefile::GetCurrentBinaryDirectory() const { - return this->StateSnapshot.GetDirectory().GetCurrentBinary(); + return this->StateSnapshot.GetDirectory().GetCurrentBinary().c_str(); } std::vector<cmTarget*> cmMakefile::GetImportedTargets() const diff --git a/Source/cmMakefileTargetGenerator.cxx b/Source/cmMakefileTargetGenerator.cxx index 7cae305..79baca6 100644 --- a/Source/cmMakefileTargetGenerator.cxx +++ b/Source/cmMakefileTargetGenerator.cxx @@ -124,7 +124,7 @@ void cmMakefileTargetGenerator::CreateRuleFile() // Open the rule file. This should be copy-if-different because the // rules may depend on this file itself. this->BuildFileStream = - new cmGeneratedFileStream(this->BuildFileNameFull.c_str(), false, + new cmGeneratedFileStream(this->BuildFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); this->BuildFileStream->SetCopyIfDifferent(true); if (!this->BuildFileStream) { @@ -237,8 +237,7 @@ void cmMakefileTargetGenerator::WriteCommonCodeRules() if (!cmSystemTools::FileExists(dependFileNameFull)) { // Write an empty dependency file. cmGeneratedFileStream depFileStream( - dependFileNameFull.c_str(), false, - this->GlobalGenerator->GetMakefileEncoding()); + dependFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); depFileStream << "# Empty dependencies file for " << this->GeneratorTarget->GetName() << ".\n" << "# This may be replaced when dependencies are built." @@ -250,7 +249,7 @@ void cmMakefileTargetGenerator::WriteCommonCodeRules() this->FlagFileNameFull = this->TargetBuildDirectoryFull; this->FlagFileNameFull += "/flags.make"; this->FlagFileStream = - new cmGeneratedFileStream(this->FlagFileNameFull.c_str(), false, + new cmGeneratedFileStream(this->FlagFileNameFull, false, this->GlobalGenerator->GetMakefileEncoding()); this->FlagFileStream->SetCopyIfDifferent(true); if (!this->FlagFileStream) { @@ -974,8 +973,7 @@ void cmMakefileTargetGenerator::WriteTargetDependRules() this->InfoFileNameFull += "/DependInfo.cmake"; this->InfoFileNameFull = this->LocalGenerator->ConvertToFullPath(this->InfoFileNameFull); - this->InfoFileStream = - new cmGeneratedFileStream(this->InfoFileNameFull.c_str()); + this->InfoFileStream = new cmGeneratedFileStream(this->InfoFileNameFull); this->InfoFileStream->SetCopyIfDifferent(true); if (!*this->InfoFileStream) { return; @@ -1446,7 +1444,7 @@ void cmMakefileTargetGenerator::CreateLinkScript( std::string linkScriptName = this->TargetBuildDirectoryFull; linkScriptName += "/"; linkScriptName += name; - cmGeneratedFileStream linkScriptStream(linkScriptName.c_str()); + cmGeneratedFileStream linkScriptStream(linkScriptName); linkScriptStream.SetCopyIfDifferent(true); for (std::string const& link_command : link_commands) { // Do not write out empty commands or commands beginning in the @@ -1530,7 +1528,7 @@ std::string cmMakefileTargetGenerator::CreateResponseFile( std::string responseFileNameFull = this->TargetBuildDirectoryFull; responseFileNameFull += "/"; responseFileNameFull += name; - cmGeneratedFileStream responseStream(responseFileNameFull.c_str()); + cmGeneratedFileStream responseStream(responseFileNameFull); responseStream.SetCopyIfDifferent(true); responseStream << options << "\n"; @@ -1718,7 +1716,7 @@ void cmMakefileTargetGenerator::GenDefFile( cmOutputConverter::SHELL); real_link_commands.insert(real_link_commands.begin(), cmd); // create a list of obj files for the -E __create_def to read - cmGeneratedFileStream fout(objlist_file.c_str()); + cmGeneratedFileStream fout(objlist_file); if (mdi->WindowsExportAllSymbols) { for (std::string const& obj : this->Objects) { diff --git a/Source/cmNinjaNormalTargetGenerator.cxx b/Source/cmNinjaNormalTargetGenerator.cxx index f94e5bc..b634b0a 100644 --- a/Source/cmNinjaNormalTargetGenerator.cxx +++ b/Source/cmNinjaNormalTargetGenerator.cxx @@ -956,7 +956,7 @@ void cmNinjaNormalTargetGenerator::WriteLinkStatement() preLinkCmdLines.push_back(std::move(cmd)); // create a list of obj files for the -E __create_def to read - cmGeneratedFileStream fout(obj_list_file.c_str()); + cmGeneratedFileStream fout(obj_list_file); if (mdi->WindowsExportAllSymbols) { cmNinjaDeps objs = this->GetObjects(); diff --git a/Source/cmNinjaTargetGenerator.cxx b/Source/cmNinjaTargetGenerator.cxx index df32f40..1cfaac5 100644 --- a/Source/cmNinjaTargetGenerator.cxx +++ b/Source/cmNinjaTargetGenerator.cxx @@ -1133,7 +1133,7 @@ void cmNinjaTargetGenerator::WriteTargetDependInfo(std::string const& lang) } std::string const tdin = this->GetTargetDependInfoPath(lang); - cmGeneratedFileStream tdif(tdin.c_str()); + cmGeneratedFileStream tdif(tdin); tdif << tdi; } diff --git a/Source/cmOutputConverter.cxx b/Source/cmOutputConverter.cxx index fd42c53..dbe6fa1 100644 --- a/Source/cmOutputConverter.cxx +++ b/Source/cmOutputConverter.cxx @@ -83,9 +83,9 @@ bool cmOutputConverter::ContainedInDirectory(std::string const& local_path, std::string const& remote_path, cmStateDirectory const& directory) { - const std::string relativePathTopBinary = + const std::string& relativePathTopBinary = directory.GetRelativePathTopBinary(); - const std::string relativePathTopSource = + const std::string& relativePathTopSource = directory.GetRelativePathTopSource(); const bool bothInBinary = diff --git a/Source/cmOutputRequiredFilesCommand.cxx b/Source/cmOutputRequiredFilesCommand.cxx index bdb98ca..e2de3f9 100644 --- a/Source/cmOutputRequiredFilesCommand.cxx +++ b/Source/cmOutputRequiredFilesCommand.cxx @@ -356,8 +356,7 @@ protected: if (!found) { // Couldn't find any dependency information. - if (this->ComplainFileRegularExpression.find( - info->IncludeName.c_str())) { + if (this->ComplainFileRegularExpression.find(info->IncludeName)) { cmSystemTools::Error("error cannot find dependencies for ", path); } else { // Destroy the name of the file so that it won't be output as a diff --git a/Source/cmQtAutoGen.h b/Source/cmQtAutoGen.h index a3e16ac..4118dc7 100644 --- a/Source/cmQtAutoGen.h +++ b/Source/cmQtAutoGen.h @@ -28,6 +28,32 @@ public: RCC }; + /// @brief Integer version + struct IntegerVersion + { + unsigned int Major = 0; + unsigned int Minor = 0; + + IntegerVersion() = default; + IntegerVersion(unsigned int major, unsigned int minor) + : Major(major) + , Minor(minor) + { + } + + bool operator>(IntegerVersion const version) + { + return (this->Major > version.Major) || + ((this->Major == version.Major) && (this->Minor > version.Minor)); + } + + bool operator>=(IntegerVersion const version) + { + return (this->Major > version.Major) || + ((this->Major == version.Major) && (this->Minor >= version.Minor)); + } + }; + public: /// @brief Returns the generator name static std::string const& GeneratorName(GeneratorT genType); diff --git a/Source/cmQtAutoGenInitializer.cxx b/Source/cmQtAutoGenInitializer.cxx index 7cfdeba..db7fde6 100644 --- a/Source/cmQtAutoGenInitializer.cxx +++ b/Source/cmQtAutoGenInitializer.cxx @@ -173,19 +173,17 @@ static bool StaticLibraryCycle(cmGeneratorTarget const* targetOrigin, return cycle; } -cmQtAutoGenInitializer::cmQtAutoGenInitializer( - cmGeneratorTarget* target, bool mocEnabled, bool uicEnabled, bool rccEnabled, - std::string const& qtVersionMajor) +cmQtAutoGenInitializer::cmQtAutoGenInitializer(cmGeneratorTarget* target, + bool mocEnabled, + bool uicEnabled, + bool rccEnabled, + IntegerVersion const& qtVersion) : Target(target) - , MultiConfig(false) - , QtVersionMajor(qtVersionMajor) + , QtVersion(qtVersion) { Moc.Enabled = mocEnabled; Uic.Enabled = uicEnabled; Rcc.Enabled = rccEnabled; - - this->QtVersionMinor = - cmQtAutoGenInitializer::GetQtMinorVersion(target, this->QtVersionMajor); } bool cmQtAutoGenInitializer::InitCustomTargets() @@ -194,6 +192,13 @@ bool cmQtAutoGenInitializer::InitCustomTargets() cmLocalGenerator* localGen = this->Target->GetLocalGenerator(); cmGlobalGenerator* globalGen = localGen->GetGlobalGenerator(); + // Configurations + this->MultiConfig = globalGen->IsMultiConfig(); + this->ConfigDefault = makefile->GetConfigurations(this->ConfigsList); + if (this->ConfigsList.empty()) { + this->ConfigsList.push_back(this->ConfigDefault); + } + // Verbosity this->Verbosity = makefile->GetSafeDefinition("CMAKE_AUTOGEN_VERBOSE"); if (!this->Verbosity.empty()) { @@ -205,115 +210,116 @@ bool cmQtAutoGenInitializer::InitCustomTargets() } } - // Configurations - this->MultiConfig = globalGen->IsMultiConfig(); - this->ConfigDefault = makefile->GetConfigurations(this->ConfigsList); - if (this->ConfigsList.empty()) { - this->ConfigsList.push_back(this->ConfigDefault); - } - - // Parallel processing - this->Parallel = this->Target->GetSafeProperty("AUTOGEN_PARALLEL"); - if (this->Parallel.empty() || (this->Parallel == "AUTO")) { - // Autodetect number of CPUs - this->Parallel = std::to_string(GetParallelCPUCount()); + // Targets FOLDER + { + const char* folder = + makefile->GetState()->GetGlobalProperty("AUTOMOC_TARGETS_FOLDER"); + if (folder == nullptr) { + folder = + makefile->GetState()->GetGlobalProperty("AUTOGEN_TARGETS_FOLDER"); + } + // Inherit FOLDER property from target (#13688) + if (folder == nullptr) { + folder = this->Target->GetProperty("FOLDER"); + } + if (folder != nullptr) { + this->TargetsFolder = folder; + } } - // Autogen target name - this->AutogenTargetName = this->Target->GetName(); - this->AutogenTargetName += "_autogen"; - - // Autogen directories + // Common directories { // Collapsed current binary directory std::string const cbd = cmSystemTools::CollapseFullPath( "", makefile->GetCurrentBinaryDirectory()); - // Autogen info dir - this->DirInfo = cbd; - this->DirInfo += makefile->GetCMakeInstance()->GetCMakeFilesDirectory(); - this->DirInfo += '/'; - this->DirInfo += this->AutogenTargetName; - this->DirInfo += ".dir"; - cmSystemTools::ConvertToUnixSlashes(this->DirInfo); - - // Autogen build dir - this->DirBuild = this->Target->GetSafeProperty("AUTOGEN_BUILD_DIR"); - if (this->DirBuild.empty()) { - this->DirBuild = cbd; - this->DirBuild += '/'; - this->DirBuild += this->AutogenTargetName; + // Info directory + this->Dir.Info = cbd; + this->Dir.Info += makefile->GetCMakeInstance()->GetCMakeFilesDirectory(); + this->Dir.Info += '/'; + this->Dir.Info += this->Target->GetName(); + this->Dir.Info += "_autogen"; + this->Dir.Info += ".dir"; + cmSystemTools::ConvertToUnixSlashes(this->Dir.Info); + + // Build directory + this->Dir.Build = this->Target->GetSafeProperty("AUTOGEN_BUILD_DIR"); + if (this->Dir.Build.empty()) { + this->Dir.Build = cbd; + this->Dir.Build += '/'; + this->Dir.Build += this->Target->GetName(); + this->Dir.Build += "_autogen"; } - cmSystemTools::ConvertToUnixSlashes(this->DirBuild); - // Remove build directories on cleanup - AddCleanFile(makefile, this->DirBuild); + cmSystemTools::ConvertToUnixSlashes(this->Dir.Build); + // Cleanup build directory + AddCleanFile(makefile, this->Dir.Build); // Working directory - this->DirWork = cbd; - cmSystemTools::ConvertToUnixSlashes(this->DirWork); + this->Dir.Work = cbd; + cmSystemTools::ConvertToUnixSlashes(this->Dir.Work); // Include directory - this->DirInclude = this->DirBuild; - this->DirInclude += "/include"; + this->Dir.Include = this->Dir.Build; + this->Dir.Include += "/include"; if (this->MultiConfig) { - this->DirInclude += "_$<CONFIG>"; + this->Dir.Include += "_$<CONFIG>"; } + // Per config include directories if (this->MultiConfig) { for (std::string const& cfg : this->ConfigsList) { - std::string& dir = this->DirConfigInclude[cfg]; - dir = this->DirBuild; + std::string& dir = this->Dir.ConfigInclude[cfg]; + dir = this->Dir.Build; dir += "/include_"; dir += cfg; } } } - // Autogen info and settings files - { - this->AutogenInfoFile = this->DirInfo; - this->AutogenInfoFile += "/AutogenInfo.cmake"; - - this->AutogenSettingsFile = this->DirInfo; - this->AutogenSettingsFile += "/AutogenOldSettings.txt"; - - if (this->MultiConfig) { - for (std::string const& cfg : this->ConfigsList) { - std::string& filename = this->AutogenConfigSettingsFile[cfg]; - filename = AppendFilenameSuffix(this->AutogenSettingsFile, "_" + cfg); - AddCleanFile(makefile, filename); - } - } else { - AddCleanFile(makefile, this->AutogenSettingsFile); - } - } - - // Autogen target FOLDER property - { - const char* folder = - makefile->GetState()->GetGlobalProperty("AUTOMOC_TARGETS_FOLDER"); - if (folder == nullptr) { - folder = - makefile->GetState()->GetGlobalProperty("AUTOGEN_TARGETS_FOLDER"); - } - // Inherit FOLDER property from target (#13688) - if (folder == nullptr) { - folder = this->Target->GetProperty("FOLDER"); - } - if (folder != nullptr) { - this->AutogenFolder = folder; - } - } - + // Moc, Uic and _autogen target settings if (this->Moc.Enabled || this->Uic.Enabled) { // Init moc specific settings if (this->Moc.Enabled && !InitMoc()) { return false; } + // Init uic specific settings if (this->Uic.Enabled && !InitUic()) { return false; } + // Autogen target name + this->AutogenTarget.Name = this->Target->GetName(); + this->AutogenTarget.Name += "_autogen"; + + // Autogen target parallel processing + this->AutogenTarget.Parallel = + this->Target->GetSafeProperty("AUTOGEN_PARALLEL"); + if (this->AutogenTarget.Parallel.empty() || + (this->AutogenTarget.Parallel == "AUTO")) { + // Autodetect number of CPUs + this->AutogenTarget.Parallel = std::to_string(GetParallelCPUCount()); + } + + // Autogen target info and settings files + { + this->AutogenTarget.InfoFile = this->Dir.Info; + this->AutogenTarget.InfoFile += "/AutogenInfo.cmake"; + + this->AutogenTarget.SettingsFile = this->Dir.Info; + this->AutogenTarget.SettingsFile += "/AutogenOldSettings.txt"; + + if (this->MultiConfig) { + for (std::string const& cfg : this->ConfigsList) { + std::string& filename = this->AutogenTarget.ConfigSettingsFile[cfg]; + filename = + AppendFilenameSuffix(this->AutogenTarget.SettingsFile, "_" + cfg); + AddCleanFile(makefile, filename); + } + } else { + AddCleanFile(makefile, this->AutogenTarget.SettingsFile); + } + } + // Autogen target: Compute user defined dependencies { std::string const deps = @@ -325,14 +331,15 @@ bool cmQtAutoGenInitializer::InitCustomTargets() // Allow target and file dependencies auto* depTarget = makefile->FindTargetToUse(depName); if (depTarget != nullptr) { - this->AutogenDependTargets.insert(depTarget); + this->AutogenTarget.DependTargets.insert(depTarget); } else { - this->AutogenDependFiles.insert(depName); + this->AutogenTarget.DependFiles.insert(depName); } } } } } + // Init rcc specific settings if (this->Rcc.Enabled && !InitRcc()) { return false; @@ -341,7 +348,7 @@ bool cmQtAutoGenInitializer::InitCustomTargets() // Add autogen include directory to the origin target INCLUDE_DIRECTORIES if (this->Moc.Enabled || this->Uic.Enabled || (this->Rcc.Enabled && this->MultiConfig)) { - this->Target->AddIncludeDirectory(this->DirInclude, true); + this->Target->AddIncludeDirectory(this->Dir.Include, true); } // Scan files @@ -368,19 +375,19 @@ bool cmQtAutoGenInitializer::InitMoc() cmLocalGenerator* localGen = this->Target->GetLocalGenerator(); // Mocs compilation file - this->Moc.MocsCompilation = this->DirBuild; + this->Moc.MocsCompilation = this->Dir.Build; this->Moc.MocsCompilation += "/mocs_compilation.cpp"; // Moc predefs command if (this->Target->GetPropertyAsBool("AUTOMOC_COMPILER_PREDEFINES") && - this->QtVersionGreaterOrEqual(5, 8)) { + (this->QtVersion >= IntegerVersion(5, 8))) { this->Moc.PredefsCmd = makefile->GetSafeDefinition("CMAKE_CXX_COMPILER_PREDEFINES_COMMAND"); } // Moc includes { - bool const appendImplicit = (this->QtVersionMajor == "5"); + bool const appendImplicit = (this->QtVersion.Major == 5); auto GetIncludeDirs = [this, localGen, appendImplicit](std::string const& cfg) -> std::string { // Get the include dirs for this target, without stripping the implicit @@ -552,15 +559,15 @@ bool cmQtAutoGenInitializer::InitScanFiles() const bool generated = sf->GetPropertyAsBool("GENERATED"); if (fileType == cmSystemTools::HEADER_FILE_FORMAT) { if (generated) { - this->MocUic.HeadersGenerated.push_back(absPath); + this->AutogenTarget.HeadersGenerated.push_back(absPath); } else { - this->MocUic.Headers.push_back(absPath); + this->AutogenTarget.Headers.push_back(absPath); } } else { if (generated) { - this->MocUic.SourcesGenerated.push_back(absPath); + this->AutogenTarget.SourcesGenerated.push_back(absPath); } else { - this->MocUic.Sources.push_back(absPath); + this->AutogenTarget.Sources.push_back(absPath); } } } @@ -631,8 +638,8 @@ bool cmQtAutoGenInitializer::InitScanFiles() } // Process GENERATED sources and headers - if (!this->MocUic.SourcesGenerated.empty() || - !this->MocUic.HeadersGenerated.empty()) { + if (!this->AutogenTarget.SourcesGenerated.empty() || + !this->AutogenTarget.HeadersGenerated.empty()) { // Check status of policy CMP0071 bool policyAccept = false; bool policyWarn = false; @@ -655,13 +662,15 @@ bool cmQtAutoGenInitializer::InitScanFiles() if (policyAccept) { // Accept GENERATED sources - for (std::string const& absFile : this->MocUic.HeadersGenerated) { - this->MocUic.Headers.push_back(absFile); - this->AutogenDependFiles.insert(absFile); + for (std::string const& absFile : + this->AutogenTarget.HeadersGenerated) { + this->AutogenTarget.Headers.push_back(absFile); + this->AutogenTarget.DependFiles.insert(absFile); } - for (std::string const& absFile : this->MocUic.SourcesGenerated) { - this->MocUic.Sources.push_back(absFile); - this->AutogenDependFiles.insert(absFile); + for (std::string const& absFile : + this->AutogenTarget.SourcesGenerated) { + this->AutogenTarget.Sources.push_back(absFile); + this->AutogenTarget.DependFiles.insert(absFile); } } else { if (policyWarn) { @@ -682,10 +691,12 @@ bool cmQtAutoGenInitializer::InitScanFiles() } msg += "For compatibility, CMake is excluding the GENERATED source " "file(s):\n"; - for (const std::string& absFile : this->MocUic.HeadersGenerated) { + for (const std::string& absFile : + this->AutogenTarget.HeadersGenerated) { msg.append(" ").append(Quoted(absFile)).append("\n"); } - for (const std::string& absFile : this->MocUic.SourcesGenerated) { + for (const std::string& absFile : + this->AutogenTarget.SourcesGenerated) { msg.append(" ").append(Quoted(absFile)).append("\n"); } msg += "from processing by "; @@ -704,14 +715,16 @@ bool cmQtAutoGenInitializer::InitScanFiles() } // Sort headers and sources if (this->Moc.Enabled || this->Uic.Enabled) { - std::sort(this->MocUic.Headers.begin(), this->MocUic.Headers.end()); - std::sort(this->MocUic.Sources.begin(), this->MocUic.Sources.end()); + std::sort(this->AutogenTarget.Headers.begin(), + this->AutogenTarget.Headers.end()); + std::sort(this->AutogenTarget.Sources.begin(), + this->AutogenTarget.Sources.end()); } } // Process qrc files if (!this->Rcc.Qrcs.empty()) { - const bool QtV5 = (this->QtVersionMajor == "5"); + const bool QtV5 = (this->QtVersion.Major == 5); // Target rcc options std::vector<std::string> optionsTarget; cmSystemTools::ExpandListArgument( @@ -734,7 +747,7 @@ bool cmQtAutoGenInitializer::InitScanFiles() qrc.PathChecksum = fpathCheckSum.getPart(qrc.QrcFile); // RCC output file name { - std::string rccFile = this->DirBuild + "/"; + std::string rccFile = this->Dir.Build + "/"; rccFile += qrc.PathChecksum; rccFile += "/qrc_"; rccFile += qrc.QrcName; @@ -742,7 +755,7 @@ bool cmQtAutoGenInitializer::InitScanFiles() qrc.RccFile = std::move(rccFile); } { - std::string base = this->DirInfo; + std::string base = this->Dir.Info; base += "/RCC"; base += qrc.QrcName; if (!qrc.Unique) { @@ -811,7 +824,7 @@ bool cmQtAutoGenInitializer::InitAutogenTarget() cmGlobalGenerator* globalGen = localGen->GetGlobalGenerator(); // Register info file as generated by CMake - makefile->AddCMakeOutputFile(this->AutogenInfoFile); + makefile->AddCMakeOutputFile(this->AutogenTarget.InfoFile); // Files provided by the autogen target std::vector<std::string> autogenProvides; @@ -846,7 +859,7 @@ bool cmQtAutoGenInitializer::InitAutogenTarget() currentLine.push_back(cmSystemTools::GetCMakeCommand()); currentLine.push_back("-E"); currentLine.push_back("cmake_autogen"); - currentLine.push_back(this->AutogenInfoFile); + currentLine.push_back(this->AutogenTarget.InfoFile); currentLine.push_back("$<CONFIGURATION>"); commandLines.push_back(std::move(currentLine)); } @@ -863,14 +876,14 @@ bool cmQtAutoGenInitializer::InitAutogenTarget() // Disable PRE_BUILD in some cases if (usePRE_BUILD) { // Cannot use PRE_BUILD with file depends - if (!this->AutogenDependFiles.empty()) { + if (!this->AutogenTarget.DependFiles.empty()) { usePRE_BUILD = false; } } // Create the autogen target/command if (usePRE_BUILD) { // Add additional autogen target dependencies to origin target - for (cmTarget* depTarget : this->AutogenDependTargets) { + for (cmTarget* depTarget : this->AutogenTarget.DependTargets) { this->Target->Target->AddUtility(depTarget->GetName(), makefile); } @@ -883,7 +896,7 @@ bool cmQtAutoGenInitializer::InitAutogenTarget() const std::vector<std::string> no_deps; cmCustomCommand cc(makefile, no_output, autogenProvides, no_deps, commandLines, autogenComment.c_str(), - this->DirWork.c_str()); + this->Dir.Work.c_str()); cc.SetEscapeOldStyle(false); cc.SetEscapeAllowMakeVars(true); this->Target->Target->AddPreBuildCommand(cc); @@ -911,17 +924,17 @@ bool cmQtAutoGenInitializer::InitAutogenTarget() } for (auto const& item : commonTargets) { if (item.second == this->ConfigsList.size()) { - this->AutogenDependTargets.insert(item.first->Target); + this->AutogenTarget.DependTargets.insert(item.first->Target); } } } // Create autogen target cmTarget* autogenTarget = makefile->AddUtilityCommand( - this->AutogenTargetName, cmMakefile::TargetOrigin::Generator, true, - this->DirWork.c_str(), /*byproducts=*/autogenProvides, - std::vector<std::string>(this->AutogenDependFiles.begin(), - this->AutogenDependFiles.end()), + this->AutogenTarget.Name, cmMakefile::TargetOrigin::Generator, true, + this->Dir.Work.c_str(), /*byproducts=*/autogenProvides, + std::vector<std::string>(this->AutogenTarget.DependFiles.begin(), + this->AutogenTarget.DependFiles.end()), commandLines, false, autogenComment.c_str()); // Create autogen generator target localGen->AddGeneratorTarget( @@ -932,17 +945,17 @@ bool cmQtAutoGenInitializer::InitAutogenTarget() autogenTarget->AddUtility(depName, makefile); } // Add additional autogen target dependencies to autogen target - for (cmTarget* depTarget : this->AutogenDependTargets) { + for (cmTarget* depTarget : this->AutogenTarget.DependTargets) { autogenTarget->AddUtility(depTarget->GetName(), makefile); } // Set FOLDER property in autogen target - if (!this->AutogenFolder.empty()) { - autogenTarget->SetProperty("FOLDER", this->AutogenFolder.c_str()); + if (!this->TargetsFolder.empty()) { + autogenTarget->SetProperty("FOLDER", this->TargetsFolder.c_str()); } // Add autogen target to the origin target dependencies - this->Target->Target->AddUtility(this->AutogenTargetName, makefile); + this->Target->Target->AddUtility(this->AutogenTarget.Name, makefile); } return true; @@ -1004,15 +1017,15 @@ bool cmQtAutoGenInitializer::InitRccTargets() cmTarget* autoRccTarget = makefile->AddUtilityCommand( ccName, cmMakefile::TargetOrigin::Generator, true, - this->DirWork.c_str(), ccOutput, ccDepends, commandLines, false, + this->Dir.Work.c_str(), ccOutput, ccDepends, commandLines, false, ccComment.c_str()); // Create autogen generator target localGen->AddGeneratorTarget( new cmGeneratorTarget(autoRccTarget, localGen)); // Set FOLDER property in autogen target - if (!this->AutogenFolder.empty()) { - autoRccTarget->SetProperty("FOLDER", this->AutogenFolder.c_str()); + if (!this->TargetsFolder.empty()) { + autoRccTarget->SetProperty("FOLDER", this->TargetsFolder.c_str()); } } // Add autogen target to the origin target dependencies @@ -1034,7 +1047,7 @@ bool cmQtAutoGenInitializer::InitRccTargets() makefile->AddCustomCommandToOutput(ccOutput, ccByproducts, ccDepends, /*main_dependency*/ std::string(), commandLines, ccComment.c_str(), - this->DirWork.c_str()); + this->Dir.Work.c_str()); } // Reconfigure when .qrc file changes makefile->AddCMakeDependFile(qrc.QrcFile); @@ -1047,9 +1060,9 @@ bool cmQtAutoGenInitializer::InitRccTargets() bool cmQtAutoGenInitializer::SetupCustomTargets() { // Create info directory on demand - if (!cmSystemTools::MakeDirectory(this->DirInfo)) { + if (!cmSystemTools::MakeDirectory(this->Dir.Info)) { std::string emsg = ("AutoGen: Could not create directory: "); - emsg += Quoted(this->DirInfo); + emsg += Quoted(this->Dir.Info); cmSystemTools::Error(emsg.c_str()); return false; } @@ -1076,13 +1089,16 @@ bool cmQtAutoGenInitializer::SetupWriteAutogenInfo() cmGeneratedFileStream ofs; ofs.SetCopyIfDifferent(true); - ofs.Open(this->AutogenInfoFile.c_str(), false, true); + ofs.Open(this->AutogenTarget.InfoFile, false, true); if (ofs) { // Utility lambdas auto CWrite = [&ofs](const char* key, std::string const& value) { ofs << "set(" << key << " " << cmOutputConverter::EscapeForCMake(value) << ")\n"; }; + auto CWriteUInt = [&ofs](const char* key, unsigned int value) { + ofs << "set(" << key << " " << value << ")\n"; + }; auto CWriteList = [&CWrite](const char* key, std::vector<std::string> const& list) { CWrite(key, cmJoin(list, ";")); @@ -1117,7 +1133,7 @@ bool cmQtAutoGenInitializer::SetupWriteAutogenInfo() // Write ofs << "# Meta\n"; CWrite("AM_MULTI_CONFIG", this->MultiConfig ? "TRUE" : "FALSE"); - CWrite("AM_PARALLEL", this->Parallel); + CWrite("AM_PARALLEL", this->AutogenTarget.Parallel); CWrite("AM_VERBOSITY", this->Verbosity); ofs << "# Directories\n"; @@ -1127,18 +1143,18 @@ bool cmQtAutoGenInitializer::SetupWriteAutogenInfo() CWrite("AM_CMAKE_CURRENT_BINARY_DIR", MfDef("CMAKE_CURRENT_BINARY_DIR")); CWrite("AM_CMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE", MfDef("CMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE")); - CWrite("AM_BUILD_DIR", this->DirBuild); - CWrite("AM_INCLUDE_DIR", this->DirInclude); - CWriteMap("AM_INCLUDE_DIR", this->DirConfigInclude); + CWrite("AM_BUILD_DIR", this->Dir.Build); + CWrite("AM_INCLUDE_DIR", this->Dir.Include); + CWriteMap("AM_INCLUDE_DIR", this->Dir.ConfigInclude); ofs << "# Files\n"; - CWriteList("AM_SOURCES", this->MocUic.Sources); - CWriteList("AM_HEADERS", this->MocUic.Headers); - CWrite("AM_SETTINGS_FILE", this->AutogenSettingsFile); - CWriteMap("AM_SETTINGS_FILE", this->AutogenConfigSettingsFile); + CWriteList("AM_SOURCES", this->AutogenTarget.Sources); + CWriteList("AM_HEADERS", this->AutogenTarget.Headers); + CWrite("AM_SETTINGS_FILE", this->AutogenTarget.SettingsFile); + CWriteMap("AM_SETTINGS_FILE", this->AutogenTarget.ConfigSettingsFile); ofs << "# Qt\n"; - CWrite("AM_QT_VERSION_MAJOR", this->QtVersionMajor); + CWriteUInt("AM_QT_VERSION_MAJOR", this->QtVersion.Major); CWrite("AM_QT_MOC_EXECUTABLE", this->Moc.Executable); CWrite("AM_QT_UIC_EXECUTABLE", this->Uic.Executable); @@ -1170,7 +1186,7 @@ bool cmQtAutoGenInitializer::SetupWriteAutogenInfo() } } else { std::string err = "AutoGen: Could not write file "; - err += this->AutogenInfoFile; + err += this->AutogenTarget.InfoFile; cmSystemTools::Error(err.c_str()); return false; } @@ -1183,7 +1199,7 @@ bool cmQtAutoGenInitializer::SetupWriteRccInfo() for (Qrc const& qrc : this->Rcc.Qrcs) { cmGeneratedFileStream ofs; ofs.SetCopyIfDifferent(true); - ofs.Open(qrc.InfoFile.c_str(), false, true); + ofs.Open(qrc.InfoFile, false, true); if (ofs) { // Utility lambdas auto CWrite = [&ofs](const char* key, std::string const& value) { @@ -1207,9 +1223,9 @@ bool cmQtAutoGenInitializer::SetupWriteRccInfo() CWriteMap("ARCC_SETTINGS_FILE", qrc.ConfigSettingsFile); ofs << "# Directories\n"; - CWrite("ARCC_BUILD_DIR", this->DirBuild); - CWrite("ARCC_INCLUDE_DIR", this->DirInclude); - CWriteMap("ARCC_INCLUDE_DIR", this->DirConfigInclude); + CWrite("ARCC_BUILD_DIR", this->Dir.Build); + CWrite("ARCC_INCLUDE_DIR", this->Dir.Include); + CWriteMap("ARCC_INCLUDE_DIR", this->Dir.ConfigInclude); ofs << "# Rcc executable\n"; CWrite("ARCC_RCC_EXECUTABLE", this->Rcc.Executable); @@ -1251,53 +1267,56 @@ void cmQtAutoGenInitializer::AddGeneratedSource(std::string const& filename, this->Target->AddSource(filename); } -std::string cmQtAutoGenInitializer::GetQtMajorVersion( +cmQtAutoGenInitializer::IntegerVersion cmQtAutoGenInitializer::GetQtVersion( cmGeneratorTarget const* target) { + cmQtAutoGenInitializer::IntegerVersion res; cmMakefile* makefile = target->Target->GetMakefile(); + + // -- Major version std::string qtMajor = makefile->GetSafeDefinition("QT_VERSION_MAJOR"); if (qtMajor.empty()) { qtMajor = makefile->GetSafeDefinition("Qt5Core_VERSION_MAJOR"); } - const char* targetQtVersion = - target->GetLinkInterfaceDependentStringProperty("QT_MAJOR_VERSION", ""); - if (targetQtVersion != nullptr) { - qtMajor = targetQtVersion; + { + const char* targetQtVersion = + target->GetLinkInterfaceDependentStringProperty("QT_MAJOR_VERSION", ""); + if (targetQtVersion != nullptr) { + qtMajor = targetQtVersion; + } } - return qtMajor; -} -std::string cmQtAutoGenInitializer::GetQtMinorVersion( - cmGeneratorTarget const* target, std::string const& qtVersionMajor) -{ - cmMakefile* makefile = target->Target->GetMakefile(); + // -- Minor version std::string qtMinor; - if (qtVersionMajor == "5") { - qtMinor = makefile->GetSafeDefinition("Qt5Core_VERSION_MINOR"); - } - if (qtMinor.empty()) { - qtMinor = makefile->GetSafeDefinition("QT_VERSION_MINOR"); + if (!qtMajor.empty()) { + if (qtMajor == "5") { + qtMinor = makefile->GetSafeDefinition("Qt5Core_VERSION_MINOR"); + } + if (qtMinor.empty()) { + qtMinor = makefile->GetSafeDefinition("QT_VERSION_MINOR"); + } + { + const char* targetQtVersion = + target->GetLinkInterfaceDependentStringProperty("QT_MINOR_VERSION", + ""); + if (targetQtVersion != nullptr) { + qtMinor = targetQtVersion; + } + } } - const char* targetQtVersion = - target->GetLinkInterfaceDependentStringProperty("QT_MINOR_VERSION", ""); - if (targetQtVersion != nullptr) { - qtMinor = targetQtVersion; + // -- Convert to integer + if (!qtMajor.empty() && !qtMinor.empty()) { + unsigned long majorUL(0); + unsigned long minorUL(0); + if (cmSystemTools::StringToULong(qtMajor.c_str(), &majorUL) && + cmSystemTools::StringToULong(qtMinor.c_str(), &minorUL)) { + res.Major = static_cast<unsigned int>(majorUL); + res.Minor = static_cast<unsigned int>(minorUL); + } } - return qtMinor; -} -bool cmQtAutoGenInitializer::QtVersionGreaterOrEqual( - unsigned long requestMajor, unsigned long requestMinor) const -{ - unsigned long majorUL(0); - unsigned long minorUL(0); - if (cmSystemTools::StringToULong(this->QtVersionMajor.c_str(), &majorUL) && - cmSystemTools::StringToULong(this->QtVersionMinor.c_str(), &minorUL)) { - return (majorUL > requestMajor) || - (majorUL == requestMajor && minorUL >= requestMinor); - } - return false; + return res; } bool cmQtAutoGenInitializer::GetMocExecutable() @@ -1307,9 +1326,9 @@ bool cmQtAutoGenInitializer::GetMocExecutable() // Find moc executable { std::string targetName; - if (this->QtVersionMajor == "5") { + if (this->QtVersion.Major == 5) { targetName = "Qt5::moc"; - } else if (QtVersionMajor == "4") { + } else if (this->QtVersion.Major == 4) { targetName = "Qt4::moc"; } else { err = "The AUTOMOC feature supports only Qt 4 and Qt 5"; @@ -1368,9 +1387,9 @@ bool cmQtAutoGenInitializer::GetUicExecutable() // Find uic executable { std::string targetName; - if (this->QtVersionMajor == "5") { + if (this->QtVersion.Major == 5) { targetName = "Qt5::uic"; - } else if (QtVersionMajor == "4") { + } else if (this->QtVersion.Major == 4) { targetName = "Qt4::uic"; } else { err = "The AUTOUIC feature supports only Qt 4 and Qt 5"; @@ -1381,7 +1400,7 @@ bool cmQtAutoGenInitializer::GetUicExecutable() if (tgt != nullptr) { this->Uic.Executable = tgt->ImportedGetLocation(""); } else { - if (this->QtVersionMajor == "5") { + if (this->QtVersion.Major == 5) { // Project does not use Qt5Widgets, but has AUTOUIC ON anyway } else { err = "Could not find target " + targetName; @@ -1433,9 +1452,9 @@ bool cmQtAutoGenInitializer::GetRccExecutable() // Find rcc executable { std::string targetName; - if (this->QtVersionMajor == "5") { + if (this->QtVersion.Major == 5) { targetName = "Qt5::rcc"; - } else if (QtVersionMajor == "4") { + } else if (this->QtVersion.Major == 4) { targetName = "Qt4::rcc"; } else { err = "The AUTORCC feature supports only Qt 4 and Qt 5"; @@ -1465,7 +1484,7 @@ bool cmQtAutoGenInitializer::GetRccExecutable() cmSystemTools::OUTPUT_NONE, cmDuration::zero(), cmProcessOutput::Auto); if (result) { // Detect if rcc supports (-)-list - if (this->QtVersionMajor == "5") { + if (this->QtVersion.Major == 5) { if (stdOut.find("--list") != std::string::npos) { this->Rcc.ListOptions.push_back("--list"); } else { diff --git a/Source/cmQtAutoGenInitializer.h b/Source/cmQtAutoGenInitializer.h index 16f9cdd..ce00e00 100644 --- a/Source/cmQtAutoGenInitializer.h +++ b/Source/cmQtAutoGenInitializer.h @@ -18,10 +18,6 @@ class cmTarget; class cmQtAutoGenInitializer : public cmQtAutoGen { public: - static std::string GetQtMajorVersion(cmGeneratorTarget const* target); - static std::string GetQtMinorVersion(cmGeneratorTarget const* target, - std::string const& qtVersionMajor); - /// @brief Rcc job information class Qrc { @@ -48,9 +44,11 @@ public: }; public: + static IntegerVersion GetQtVersion(cmGeneratorTarget const* target); + cmQtAutoGenInitializer(cmGeneratorTarget* target, bool mocEnabled, bool uicEnabled, bool rccEnabled, - std::string const& qtVersionMajor); + IntegerVersion const& qtVersion); bool InitCustomTargets(); bool SetupCustomTargets(); @@ -69,9 +67,6 @@ private: void AddGeneratedSource(std::string const& filename, GeneratorT genType); - bool QtVersionGreaterOrEqual(unsigned long requestMajor, - unsigned long requestMinor) const; - bool GetMocExecutable(); bool GetUicExecutable(); bool GetRccExecutable(); @@ -82,39 +77,46 @@ private: private: cmGeneratorTarget* Target; + + // Configuration + IntegerVersion QtVersion; bool MultiConfig = false; - // Qt - std::string QtVersionMajor; - std::string QtVersionMinor; - // Configurations std::string ConfigDefault; std::vector<std::string> ConfigsList; - std::string Parallel; std::string Verbosity; - // Names - std::string AutogenTargetName; - std::string AutogenFolder; - std::string AutogenInfoFile; - std::string AutogenSettingsFile; - std::map<std::string, std::string> AutogenConfigSettingsFile; - std::set<std::string> AutogenDependFiles; - std::set<cmTarget*> AutogenDependTargets; - // Directories - std::string DirInfo; - std::string DirBuild; - std::string DirWork; - std::string DirInclude; - std::map<std::string, std::string> DirConfigInclude; - // Moc and UIC + std::string TargetsFolder; + + /// @brief Common directories + struct + { + std::string Info; + std::string Build; + std::string Work; + std::string Include; + std::map<std::string, std::string> ConfigInclude; + } Dir; + + /// @brief Autogen target variables struct { - // Sources + std::string Name; + // Settings + std::string Parallel; + // Configuration files + std::string InfoFile; + std::string SettingsFile; + std::map<std::string, std::string> ConfigSettingsFile; + // Dependencies + std::set<std::string> DependFiles; + std::set<cmTarget*> DependTargets; + // Sources to process std::vector<std::string> Headers; std::vector<std::string> Sources; std::vector<std::string> HeadersGenerated; std::vector<std::string> SourcesGenerated; - } MocUic; - // Moc + } AutogenTarget; + + /// @brief Moc only variables struct { bool Enabled = false; @@ -127,7 +129,8 @@ private: std::map<std::string, std::string> ConfigDefines; std::string MocsCompilation; } Moc; - // Uic + + ///@brief Uic only variables struct { bool Enabled = false; @@ -139,7 +142,8 @@ private: std::vector<std::string> FileFiles; std::vector<std::vector<std::string>> FileOptions; } Uic; - // Rcc + + /// @brief Rcc only variables struct { bool Enabled = false; diff --git a/Source/cmServerProtocol.cxx b/Source/cmServerProtocol.cxx index b05eac4..f0a5e26 100644 --- a/Source/cmServerProtocol.cxx +++ b/Source/cmServerProtocol.cxx @@ -37,7 +37,6 @@ #include <set> #include <string> #include <unordered_map> -#include <utility> #include <vector> // Get rid of some windows macros: @@ -798,9 +797,8 @@ static Json::Value DumpCTestInfo(cmLocalGenerator* lg, cmTest* testInfo, // Remove any config specific variables from the output. cmGeneratorExpression ge; - auto cge = ge.Parse(command.c_str()); - const char* processed = cge->Evaluate(lg, config); - + auto cge = ge.Parse(command); + const std::string& processed = cge->Evaluate(lg, config); result[kCTEST_COMMAND] = processed; // Build up the list of properties that may have been specified @@ -811,7 +809,7 @@ static Json::Value DumpCTestInfo(cmLocalGenerator* lg, cmTest* testInfo, // Remove config variables from the value too. auto cge_value = ge.Parse(prop.second.GetValue()); - const char* processed_value = cge_value->Evaluate(lg, config); + const std::string& processed_value = cge_value->Evaluate(lg, config); entry[kVALUE_KEY] = processed_value; properties.append(entry); } diff --git a/Source/cmState.cxx b/Source/cmState.cxx index dcf6ea0..5651594 100644 --- a/Source/cmState.cxx +++ b/Source/cmState.cxx @@ -267,6 +267,7 @@ cmStateSnapshot cmState::Reset() { this->GlobalProperties.clear(); this->PropertyDefinitions.clear(); + this->GlobVerificationManager->Reset(); cmStateDetail::PositionType pos = this->SnapshotData.Truncate(); this->ExecutionListFiles.Truncate(); diff --git a/Source/cmStateDirectory.cxx b/Source/cmStateDirectory.cxx index 6eac8e2..925b161 100644 --- a/Source/cmStateDirectory.cxx +++ b/Source/cmStateDirectory.cxx @@ -84,9 +84,9 @@ void cmStateDirectory::ComputeRelativePathTopBinary() } } -const char* cmStateDirectory::GetCurrentSource() const +std::string const& cmStateDirectory::GetCurrentSource() const { - return this->DirectoryState->Location.c_str(); + return this->DirectoryState->Location; } void cmStateDirectory::SetCurrentSource(std::string const& dir) @@ -101,9 +101,9 @@ void cmStateDirectory::SetCurrentSource(std::string const& dir) this->Snapshot_.SetDefinition("CMAKE_CURRENT_SOURCE_DIR", loc); } -const char* cmStateDirectory::GetCurrentBinary() const +std::string const& cmStateDirectory::GetCurrentBinary() const { - return this->DirectoryState->OutputLocation.c_str(); + return this->DirectoryState->OutputLocation; } void cmStateDirectory::SetCurrentBinary(std::string const& dir) @@ -118,14 +118,14 @@ void cmStateDirectory::SetCurrentBinary(std::string const& dir) this->Snapshot_.SetDefinition("CMAKE_CURRENT_BINARY_DIR", loc); } -const char* cmStateDirectory::GetRelativePathTopSource() const +std::string const& cmStateDirectory::GetRelativePathTopSource() const { - return this->DirectoryState->RelativePathTopSource.c_str(); + return this->DirectoryState->RelativePathTopSource; } -const char* cmStateDirectory::GetRelativePathTopBinary() const +std::string const& cmStateDirectory::GetRelativePathTopBinary() const { - return this->DirectoryState->RelativePathTopBinary.c_str(); + return this->DirectoryState->RelativePathTopBinary; } void cmStateDirectory::SetRelativePathTopSource(const char* dir) @@ -474,7 +474,7 @@ const char* cmStateDirectory::GetProperty(const std::string& prop, if (prop == "PARENT_DIRECTORY") { cmStateSnapshot parent = this->Snapshot_.GetBuildsystemDirectoryParent(); if (parent.IsValid()) { - return parent.GetDirectory().GetCurrentSource(); + return parent.GetDirectory().GetCurrentSource().c_str(); } return ""; } diff --git a/Source/cmStateDirectory.h b/Source/cmStateDirectory.h index bc96cc9..412664f 100644 --- a/Source/cmStateDirectory.h +++ b/Source/cmStateDirectory.h @@ -22,13 +22,13 @@ class cmStateDirectory cmStateSnapshot const& snapshot); public: - const char* GetCurrentSource() const; + std::string const& GetCurrentSource() const; void SetCurrentSource(std::string const& dir); - const char* GetCurrentBinary() const; + std::string const& GetCurrentBinary() const; void SetCurrentBinary(std::string const& dir); - const char* GetRelativePathTopSource() const; - const char* GetRelativePathTopBinary() const; + std::string const& GetRelativePathTopSource() const; + std::string const& GetRelativePathTopBinary() const; void SetRelativePathTopSource(const char* dir); void SetRelativePathTopBinary(const char* dir); diff --git a/Source/cmStringCommand.cxx b/Source/cmStringCommand.cxx index ed88a1e..1605fd7 100644 --- a/Source/cmStringCommand.cxx +++ b/Source/cmStringCommand.cxx @@ -273,7 +273,7 @@ bool cmStringCommand::RegexMatch(std::vector<std::string> const& args) // Scan through the input for all matches. std::string output; - if (re.find(input.c_str())) { + if (re.find(input)) { this->Makefile->StoreMatches(re); std::string::size_type l = re.start(); std::string::size_type r = re.end(); diff --git a/Source/cmTarget.cxx b/Source/cmTarget.cxx index 380ac88..cc5a176 100644 --- a/Source/cmTarget.cxx +++ b/Source/cmTarget.cxx @@ -1415,13 +1415,15 @@ const char* cmTarget::GetProperty(const std::string& prop) const return this->GetMakefile() ->GetStateSnapshot() .GetDirectory() - .GetCurrentBinary(); + .GetCurrentBinary() + .c_str(); } if (prop == propSOURCE_DIR) { return this->GetMakefile() ->GetStateSnapshot() .GetDirectory() - .GetCurrentSource(); + .GetCurrentSource() + .c_str(); } } diff --git a/Source/cmTestGenerator.cxx b/Source/cmTestGenerator.cxx index c532efb..796d2df 100644 --- a/Source/cmTestGenerator.cxx +++ b/Source/cmTestGenerator.cxx @@ -103,7 +103,7 @@ void cmTestGenerator::GenerateScriptForConfig(std::ostream& os, } } else { // Use the command name given. - exe = ge.Parse(exe.c_str())->Evaluate(this->LG, config); + exe = ge.Parse(exe)->Evaluate(this->LG, config); cmSystemTools::ConvertToUnixSlashes(exe); } diff --git a/Source/cmUseMangledMesaCommand.cxx b/Source/cmUseMangledMesaCommand.cxx index ea012f6..01ef5cb 100644 --- a/Source/cmUseMangledMesaCommand.cxx +++ b/Source/cmUseMangledMesaCommand.cxx @@ -82,12 +82,12 @@ void cmUseMangledMesaCommand::CopyAndFullPathMesaHeader(const char* source, // regular expression for gl GL or xmesa in a file (match(1) of above) cmsys::RegularExpression glLine("(gl|GL|xmesa)"); while (cmSystemTools::GetLineFromStream(fin, inLine)) { - if (includeLine.find(inLine.c_str())) { + if (includeLine.find(inLine)) { std::string includeFile = includeLine.match(1); - if (glDirLine.find(includeFile.c_str())) { + if (glDirLine.find(includeFile)) { std::string gfile = glDirLine.match(3); fout << "#include \"" << outdir << "/" << gfile << "\"\n"; - } else if (glLine.find(includeFile.c_str())) { + } else if (glLine.find(includeFile)) { fout << "#include \"" << outdir << "/" << includeLine.match(1) << "\"\n"; } else { diff --git a/Source/cmVisualStudio10TargetGenerator.cxx b/Source/cmVisualStudio10TargetGenerator.cxx index 41567a3..aa43457 100644 --- a/Source/cmVisualStudio10TargetGenerator.cxx +++ b/Source/cmVisualStudio10TargetGenerator.cxx @@ -1742,10 +1742,8 @@ void cmVisualStudio10TargetGenerator::WriteExtraSource(Elem& e1, e2.Element("Link", deployLocation + "\\%(FileName)%(Extension)"); } for (size_t i = 0; i != this->Configurations.size(); ++i) { - if (0 == - strcmp( - cge->Evaluate(this->LocalGenerator, this->Configurations[i]), - "1")) { + if (cge->Evaluate(this->LocalGenerator, this->Configurations[i]) == + "1") { e2.WritePlatformConfigTag("DeploymentContent", "'$(Configuration)|$(Platform)'=='" + this->Configurations[i] + "|" + @@ -1793,14 +1791,14 @@ void cmVisualStudio10TargetGenerator::WriteExtraSource(Elem& e1, ge.Parse(shaderEnableDebug); for (size_t i = 0; i != this->Configurations.size(); ++i) { - const char* enableDebug = + const std::string& enableDebug = cge->Evaluate(this->LocalGenerator, this->Configurations[i]); - if (strlen(enableDebug) > 0) { + if (!enableDebug.empty()) { e2.WritePlatformConfigTag( "EnableDebuggingInformation", "'$(Configuration)|$(Platform)'=='" + this->Configurations[i] + "|" + this->Platform + "'", - cmSystemTools::IsOn(enableDebug) ? "true" : "false"); + cmSystemTools::IsOn(enableDebug.c_str()) ? "true" : "false"); } } } @@ -1810,14 +1808,15 @@ void cmVisualStudio10TargetGenerator::WriteExtraSource(Elem& e1, ge.Parse(shaderDisableOptimizations); for (size_t i = 0; i != this->Configurations.size(); ++i) { - const char* disableOptimizations = + const std::string& disableOptimizations = cge->Evaluate(this->LocalGenerator, this->Configurations[i]); - if (strlen(disableOptimizations) > 0) { + if (!disableOptimizations.empty()) { e2.WritePlatformConfigTag( "DisableOptimizations", "'$(Configuration)|$(Platform)'=='" + this->Configurations[i] + "|" + this->Platform + "'", - (cmSystemTools::IsOn(disableOptimizations) ? "true" : "false")); + (cmSystemTools::IsOn(disableOptimizations.c_str()) ? "true" + : "false")); } } } @@ -2129,7 +2128,7 @@ void cmVisualStudio10TargetGenerator::OutputSourceSpecificFlags( if (configDependentFlags) { clOptions.Parse(genexInterpreter.Evaluate(flags, "COMPILE_FLAGS")); } else { - clOptions.Parse(flags.c_str()); + clOptions.Parse(flags); } if (!options.empty()) { std::string expandedOptions; @@ -2140,7 +2139,7 @@ void cmVisualStudio10TargetGenerator::OutputSourceSpecificFlags( } else { this->LocalGenerator->AppendCompileOptions(expandedOptions, options); } - clOptions.Parse(expandedOptions.c_str()); + clOptions.Parse(expandedOptions); } if (clOptions.HasFlag("DisableSpecificWarnings")) { clOptions.AppendFlag("DisableSpecificWarnings", @@ -2491,6 +2490,16 @@ bool cmVisualStudio10TargetGenerator::ComputeClOptions( if (this->MSTools) { if (this->ProjectType == vcxproj) { clOptions.FixExceptionHandlingDefault(); + if (this->GlobalGenerator->GetVersion() >= + cmGlobalVisualStudioGenerator::VS15) { + // Toolsets that come with VS 2017 may now enable UseFullPaths + // by default and there is no negative /FC option that projects + // can use to switch it back. Older toolsets disable this by + // default anyway so this will not hurt them. If the project + // is using an explicit /FC option then parsing flags will + // replace this setting with "true" below. + clOptions.AddFlag("UseFullPaths", "false"); + } clOptions.AddFlag("PrecompiledHeader", "NotUsing"); std::string asmLocation = configName + "/"; clOptions.AddFlag("AssemblerListingLocation", asmLocation); @@ -2524,8 +2533,8 @@ bool cmVisualStudio10TargetGenerator::ComputeClOptions( } } - clOptions.Parse(flags.c_str()); - clOptions.Parse(defineFlags.c_str()); + clOptions.Parse(flags); + clOptions.Parse(defineFlags); std::vector<std::string> targetDefines; switch (this->ProjectType) { case vcxproj: @@ -2692,7 +2701,7 @@ bool cmVisualStudio10TargetGenerator::ComputeRcOptions( std::string(" ") + std::string(this->Makefile->GetSafeDefinition(rcConfigFlagsVar)); - rcOptions.Parse(flags.c_str()); + rcOptions.Parse(flags); // For historical reasons, add the C preprocessor defines to RC. Options& clOptions = *(this->ClOptions[configName]); @@ -2754,8 +2763,8 @@ bool cmVisualStudio10TargetGenerator::ComputeCudaOptions( // Get preprocessor definitions for this directory. std::string defineFlags = this->Makefile->GetDefineFlags(); - cudaOptions.Parse(flags.c_str()); - cudaOptions.Parse(defineFlags.c_str()); + cudaOptions.Parse(flags); + cudaOptions.Parse(defineFlags); cudaOptions.ParseFinish(); // If we haven't explicitly enabled GPU debug information @@ -2962,7 +2971,7 @@ bool cmVisualStudio10TargetGenerator::ComputeMasmOptions( std::string(" ") + std::string(this->Makefile->GetSafeDefinition(configFlagsVar)); - masmOptions.Parse(flags.c_str()); + masmOptions.Parse(flags); // Get includes for this target masmOptions.AddIncludes(this->GetIncludes(configName, "ASM_MASM")); @@ -3019,7 +3028,7 @@ bool cmVisualStudio10TargetGenerator::ComputeNasmOptions( this->Makefile->GetSafeDefinition("CMAKE_ASM_NASM_OBJECT_FORMAT")) + std::string(" ") + std::string(this->Makefile->GetSafeDefinition(configFlagsVar)); - nasmOptions.Parse(flags.c_str()); + nasmOptions.Parse(flags); // Get includes for this target nasmOptions.AddIncludes(this->GetIncludes(configName, "ASM_NASM")); @@ -3065,7 +3074,7 @@ void cmVisualStudio10TargetGenerator::WriteLibOptions( cmVS10GeneratorOptions libOptions(this->LocalGenerator, cmVisualStudioGeneratorOptions::Linker, gg->GetLibFlagTable(), this); - libOptions.Parse(libflags.c_str()); + libOptions.Parse(libflags); OptionsHelper oh(libOptions, e2); oh.PrependInheritedString("AdditionalOptions"); oh.OutputFlagMap(); @@ -3411,7 +3420,7 @@ bool cmVisualStudio10TargetGenerator::ComputeLinkOptions( linkOptions.AddFlag("SoName", targetNameSO); } - linkOptions.Parse(flags.c_str()); + linkOptions.Parse(flags); linkOptions.FixManifestUACFlags(); if (this->MSTools) { diff --git a/Source/cmVisualStudioGeneratorOptions.cxx b/Source/cmVisualStudioGeneratorOptions.cxx index 4afef8b..5d67dcf 100644 --- a/Source/cmVisualStudioGeneratorOptions.cxx +++ b/Source/cmVisualStudioGeneratorOptions.cxx @@ -297,12 +297,12 @@ void cmVisualStudioGeneratorOptions::FixManifestUACFlags() AddFlag(ENABLE_UAC, "true"); } -void cmVisualStudioGeneratorOptions::Parse(const char* flags) +void cmVisualStudioGeneratorOptions::Parse(const std::string& flags) { // Parse the input string as a windows command line since the string // is intended for writing directly into the build files. std::vector<std::string> args; - cmSystemTools::ParseWindowsCommandLine(flags, args); + cmSystemTools::ParseWindowsCommandLine(flags.c_str(), args); // Process flags that need to be represented specially in the IDE // project file. @@ -366,7 +366,7 @@ void cmVisualStudioGeneratorOptions::Reparse(std::string const& key) std::string const original = i->second[0]; i->second[0] = ""; this->UnknownFlagField = key; - this->Parse(original.c_str()); + this->Parse(original); } void cmVisualStudioGeneratorOptions::StoreUnknownFlag(std::string const& flag) diff --git a/Source/cmVisualStudioGeneratorOptions.h b/Source/cmVisualStudioGeneratorOptions.h index c6b594d..a30a67f 100644 --- a/Source/cmVisualStudioGeneratorOptions.h +++ b/Source/cmVisualStudioGeneratorOptions.h @@ -42,7 +42,7 @@ public: void ClearTables(); // Store options from command line flags. - void Parse(const char* flags); + void Parse(const std::string& flags); void ParseFinish(); void PrependInheritedString(std::string const& key); diff --git a/Source/cmake.cxx b/Source/cmake.cxx index 6edaa74..5bb1f22 100644 --- a/Source/cmake.cxx +++ b/Source/cmake.cxx @@ -407,7 +407,7 @@ bool cmake::SetCacheArgs(const std::vector<std::string>& args) for (std::string const& ck : cacheKeys) { cmStateEnums::CacheEntryType t = this->State->GetCacheEntryType(ck); if (t != cmStateEnums::STATIC) { - if (regex.find(ck.c_str())) { + if (regex.find(ck)) { entriesToDelete.push_back(ck); } } diff --git a/Source/cmcmd.cxx b/Source/cmcmd.cxx index 0a75e77..2027722 100644 --- a/Source/cmcmd.cxx +++ b/Source/cmcmd.cxx @@ -24,6 +24,7 @@ #if defined(CMAKE_BUILD_WITH_CMAKE) && defined(_WIN32) # include "bindexplib.h" +# include "cmsys/ConsoleBuf.hxx" #endif #if defined(CMAKE_BUILD_WITH_CMAKE) && defined(_WIN32) && !defined(__CYGWIN__) @@ -1545,6 +1546,15 @@ private: // still works. int cmcmd::VisualStudioLink(std::vector<std::string> const& args, int type) { +#if defined(_WIN32) && defined(CMAKE_BUILD_WITH_CMAKE) + // Replace streambuf so we output in the system codepage. CMake is set up + // to output in Unicode (see SetUTF8Pipes) but the Visual Studio linker + // outputs using the system codepage so we need to change behavior when + // we run the link command. + cmsys::ConsoleBuf::Manager consoleOut(std::cout); + cmsys::ConsoleBuf::Manager consoleErr(std::cerr, true); +#endif + if (args.size() < 2) { return -1; } @@ -1775,6 +1785,8 @@ int cmVSLink::LinkIncremental() if (!fout) { return -1; } + // Insert a pragma statement to specify utf-8 encoding. + fout << "#pragma code_page(65001)\n"; fout << this->Type << " /* CREATEPROCESS_MANIFEST_RESOURCE_ID */ " "24 /* RT_MANIFEST */ \"" diff --git a/Source/kwsys/SystemTools.cxx b/Source/kwsys/SystemTools.cxx index d552d9f..476fe08 100644 --- a/Source/kwsys/SystemTools.cxx +++ b/Source/kwsys/SystemTools.cxx @@ -1197,9 +1197,27 @@ bool SystemTools::FileExists(const std::string& filename) } return access(filename.c_str(), R_OK) == 0; #elif defined(_WIN32) - return ( - GetFileAttributesW(Encoding::ToWindowsExtendedPath(filename).c_str()) != - INVALID_FILE_ATTRIBUTES); + DWORD attr = + GetFileAttributesW(Encoding::ToWindowsExtendedPath(filename).c_str()); + if (attr == INVALID_FILE_ATTRIBUTES) { + return false; + } + + if (attr & FILE_ATTRIBUTE_REPARSE_POINT) { + // Using 0 instead of GENERIC_READ as it allows reading of file attributes + // even if we do not have permission to read the file itself + HANDLE handle = + CreateFileW(Encoding::ToWindowsExtendedPath(filename).c_str(), 0, 0, + NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + + if (handle == INVALID_HANDLE_VALUE) { + return false; + } + + CloseHandle(handle); + } + + return true; #else // SCO OpenServer 5.0.7/3.2's command has 711 permission. # if defined(_SCO_DS) diff --git a/Tests/CMakeCommands/target_link_options/CMakeLists.txt b/Tests/CMakeCommands/target_link_options/CMakeLists.txt index c66cd37..3bb6ff3 100644 --- a/Tests/CMakeCommands/target_link_options/CMakeLists.txt +++ b/Tests/CMakeCommands/target_link_options/CMakeLists.txt @@ -15,5 +15,12 @@ if (NOT result MATCHES "-PRIVATE_FLAG") endif() get_target_property(result target_link_options_2 INTERFACE_LINK_OPTIONS) if (NOT result MATCHES "-INTERFACE_FLAG") - message(SEND_ERROR "target_link_options not populated the INTERFACE_LINK_OPTIONS target property") + message(SEND_ERROR "target_link_options not populated the INTERFACE_LINK_OPTIONS target property of shared library") +endif() + +add_library(target_link_options_3 STATIC EXCLUDE_FROM_ALL LinkOptionsLib.c) +target_link_options(target_link_options_3 INTERFACE -INTERFACE_FLAG) +get_target_property(result target_link_options_3 INTERFACE_LINK_OPTIONS) +if (NOT result MATCHES "-INTERFACE_FLAG") + message(SEND_ERROR "target_link_options not populated the INTERFACE_LINK_OPTIONS target property of static library") endif() diff --git a/Tests/CudaOnly/LinkSystemDeviceLibraries/main.cu b/Tests/CudaOnly/LinkSystemDeviceLibraries/main.cu index 7eecec1..2c7c388 100644 --- a/Tests/CudaOnly/LinkSystemDeviceLibraries/main.cu +++ b/Tests/CudaOnly/LinkSystemDeviceLibraries/main.cu @@ -3,6 +3,9 @@ #include <cuda_runtime.h> #include <iostream> +// this test only makes sense for versions of CUDA that ships +// static libraries that have separable compilation device symbols +#if __CUDACC_VER_MAJOR__ <= 9 __global__ void deviceCublasSgemm(int n, float alpha, float beta, const float* d_A, const float* d_B, float* d_C) @@ -22,6 +25,7 @@ __global__ void deviceCublasSgemm(int n, float alpha, float beta, cublasDestroy(cnpHandle); } +#endif int choose_cuda_device() { @@ -63,6 +67,7 @@ int main(int argc, char** argv) return 0; } +#if __CUDACC_VER_MAJOR__ <= 9 // initial values that will make sure that the cublasSgemm won't actually // do any work int n = 0; @@ -72,6 +77,7 @@ int main(int argc, char** argv) float* d_B = nullptr; float* d_C = nullptr; deviceCublasSgemm<<<1, 1>>>(n, alpha, beta, d_A, d_B, d_C); +#endif return 0; } diff --git a/Tests/FindPackageTest/CMakeLists.txt b/Tests/FindPackageTest/CMakeLists.txt index 0876871..3fd5541 100644 --- a/Tests/FindPackageTest/CMakeLists.txt +++ b/Tests/FindPackageTest/CMakeLists.txt @@ -188,7 +188,7 @@ find_package(ArchC 3.1 EXACT NAMES zot) find_package(ArchD 4.0 EXACT NAMES zot) unset(CMAKE_LIBRARY_ARCHITECTURE) -# Test <Package>_DIR environment variable. +# Test <PackageName>_DIR environment variable. # We erase the main prefix path to ensure the env var is used. set(CMAKE_PREFIX_PATH) set(ENV{EnvA_DIR} "${CMAKE_CURRENT_SOURCE_DIR}/lib/zot-3.1") diff --git a/Tests/QtAutogen/AutogenTest.cmake b/Tests/QtAutogen/AutogenTest.cmake index 8c0a14f..3969a89 100644 --- a/Tests/QtAutogen/AutogenTest.cmake +++ b/Tests/QtAutogen/AutogenTest.cmake @@ -6,7 +6,7 @@ if(QT_QMAKE_EXECUTABLE) list(APPEND CMAKE_PREFIX_PATH ${Qt_PREFIX_DIR}) endif() -if (QT_TEST_VERSION STREQUAL 4) +if (QT_TEST_VERSION EQUAL 4) find_package(Qt4 REQUIRED) include(UseQt4) @@ -21,7 +21,7 @@ if (QT_TEST_VERSION STREQUAL 4) qt4_generate_moc(${ARGN}) endmacro() -elseif(QT_TEST_VERSION STREQUAL 5) +elseif(QT_TEST_VERSION EQUAL 5) find_package(Qt5Widgets REQUIRED) set(QT_QTCORE_TARGET Qt5::Core) diff --git a/Tests/QtAutogen/CommonTests.cmake b/Tests/QtAutogen/CommonTests.cmake index 55d36e6..01ed7e9 100644 --- a/Tests/QtAutogen/CommonTests.cmake +++ b/Tests/QtAutogen/CommonTests.cmake @@ -12,7 +12,7 @@ if(QT_TEST_ALLOW_QT_MACROS) endif() ADD_AUTOGEN_TEST(UicSkipSource) ADD_AUTOGEN_TEST(RccSkipSource) -if(NOT QT_TEST_VERSION STREQUAL 4) +if(QT_TEST_VERSION GREATER 4) ADD_AUTOGEN_TEST(MocMacroName mocMacroName) endif() ADD_AUTOGEN_TEST(MocDepends) @@ -23,13 +23,13 @@ endif() if(QT_TEST_ALLOW_QT_MACROS) ADD_AUTOGEN_TEST(MocCMP0071) endif() -if(NOT QT_TEST_VERSION STREQUAL 4) - ADD_AUTOGEN_TEST(MocOsMacros mocOsMacros) +if(QT_TEST_VERSION GREATER 4) + ADD_AUTOGEN_TEST(MocOsMacros) endif() ADD_AUTOGEN_TEST(UicInclude uicInclude) ADD_AUTOGEN_TEST(UicInterface QtAutoUicInterface) ADD_AUTOGEN_TEST(ObjectLibrary someProgram) -if(APPLE AND (NOT QT_TEST_VERSION STREQUAL 4)) +if(APPLE AND (QT_TEST_VERSION GREATER 4)) ADD_AUTOGEN_TEST(MacOsFW) endif() ADD_AUTOGEN_TEST(Parallel parallel) @@ -42,7 +42,7 @@ ADD_AUTOGEN_TEST(SameName sameName) ADD_AUTOGEN_TEST(StaticLibraryCycle slc) # Rerun tests ADD_AUTOGEN_TEST(RerunMocBasic) -if(NOT QT_TEST_VERSION STREQUAL 4) +if(QT_TEST_VERSION GREATER 4) ADD_AUTOGEN_TEST(RerunMocPlugin) endif() ADD_AUTOGEN_TEST(RerunRccDepends) diff --git a/Tests/QtAutogen/DefinesTest/CMakeLists.txt b/Tests/QtAutogen/DefinesTest/CMakeLists.txt index de22845..3761dd8 100644 --- a/Tests/QtAutogen/DefinesTest/CMakeLists.txt +++ b/Tests/QtAutogen/DefinesTest/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.10) project(DefinesTest) # Qt4 only definitions test -if(NOT QT_TEST_VERSION STREQUAL 4) +if(NOT QT_TEST_VERSION EQUAL 4) message(ERROR "Invalid Qt test version. This test is for Qt4 only.") endif() diff --git a/Tests/QtAutogen/MocOsMacros/CMakeLists.txt b/Tests/QtAutogen/MocOsMacros/CMakeLists.txt index 51a0ebd..e7b670e 100644 --- a/Tests/QtAutogen/MocOsMacros/CMakeLists.txt +++ b/Tests/QtAutogen/MocOsMacros/CMakeLists.txt @@ -2,15 +2,31 @@ cmake_minimum_required(VERSION 3.11) project(MocOsMacros) include("../AutogenTest.cmake") -# Tests if moc processes the macro Q_OS_MAC +# Tests if moc processes Q_OS_XXX macros -set(CMAKE_AUTOMOC True) +message( "Qt5Core_VERSION: ${Qt5Core_VERSION}" ) +message( + "CMAKE_CXX_COMPILER_PREDEFINES_COMMAND: " + ${CMAKE_CXX_COMPILER_PREDEFINES_COMMAND} ) -add_executable(mocOsMacros - main.cpp - TestClass.cpp - TestClass.hpp +# On some platforms (e.g. MAC) Q_OS_XXX requires moc to include moc_predefs.h +# which is supported since Qt 5.8 and requires +# CMAKE_CXX_COMPILER_PREDEFINES_COMMAND to be defined. +if( ( ${Qt5Core_VERSION} VERSION_GREATER_EQUAL "5.8" ) AND + DEFINED CMAKE_CXX_COMPILER_PREDEFINES_COMMAND ) -target_link_libraries(mocOsMacros PRIVATE ${QT_QTCORE_TARGET}) + message( "Test enabled!" ) + message( + "CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES: " + ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES} ) -message ( "CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES: ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES}" ) + set(CMAKE_AUTOMOC True) + add_executable(mocOsMacros + main.cpp + TestClass.cpp + TestClass.hpp + ) + target_link_libraries(mocOsMacros PRIVATE ${QT_QTCORE_TARGET}) +else() + message( "Test disabled!" ) +endif() diff --git a/Tests/QtAutogen/RerunMocPlugin/MocPlugin/CMakeLists.txt b/Tests/QtAutogen/RerunMocPlugin/MocPlugin/CMakeLists.txt index bc0085f..ca22aeb 100644 --- a/Tests/QtAutogen/RerunMocPlugin/MocPlugin/CMakeLists.txt +++ b/Tests/QtAutogen/RerunMocPlugin/MocPlugin/CMakeLists.txt @@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.10) project(MocPlugin) include("../../AutogenTest.cmake") -if (NOT QT_TEST_VERSION STREQUAL 5) - message(SEND_ERROR "Invalid Qt version specified.") +if (QT_TEST_VERSION LESS 5) + message(SEND_ERROR "Qt 5 or higher required.") endif() set(CMAKE_AUTOMOC_DEPEND_FILTERS diff --git a/Tests/QtAutogen/SameName/CMakeLists.txt b/Tests/QtAutogen/SameName/CMakeLists.txt index c7d6e52..931e40f 100644 --- a/Tests/QtAutogen/SameName/CMakeLists.txt +++ b/Tests/QtAutogen/SameName/CMakeLists.txt @@ -28,7 +28,7 @@ set_target_properties(sameName PROPERTIES ) # Set different compression levels -if (QT_TEST_VERSION STREQUAL 4) +if (QT_TEST_VERSION EQUAL 4) set(rccCompress "-compress") else() set(rccCompress "--compress") diff --git a/Tests/QtAutogen/TestMacros.cmake b/Tests/QtAutogen/TestMacros.cmake index 8e4bea2..0e27188 100644 --- a/Tests/QtAutogen/TestMacros.cmake +++ b/Tests/QtAutogen/TestMacros.cmake @@ -50,7 +50,7 @@ if(NON_ASCII_BDIR) # Qt4 moc does not support utf8 paths in _parameter files generated by # qtx_wrap_cpp # https://bugreports.qt.io/browse/QTBUG-35480 - if(QT_TEST_VERSION STREQUAL 4) + if(QT_TEST_VERSION EQUAL 4) set(QT_TEST_ALLOW_QT_MACROS FALSE) endif() # On windows qtx_wrap_cpp also fails in Qt5 when used on a path that diff --git a/Tests/RunCMake/Syntax/QueryCache-stderr.txt b/Tests/RunCMake/Syntax/QueryCache-stderr.txt new file mode 100644 index 0000000..db6b2b7 --- /dev/null +++ b/Tests/RunCMake/Syntax/QueryCache-stderr.txt @@ -0,0 +1,2 @@ +-->cache value<-- +-->local value<-- diff --git a/Tests/RunCMake/Syntax/QueryCache.cmake b/Tests/RunCMake/Syntax/QueryCache.cmake new file mode 100644 index 0000000..20c648e --- /dev/null +++ b/Tests/RunCMake/Syntax/QueryCache.cmake @@ -0,0 +1,6 @@ + +set(query_var "cache value" CACHE STRING "") +set(query_var "local value") + +message("-->$CACHE{query_var}<--") +message("-->${query_var}<--") diff --git a/Tests/RunCMake/Syntax/RunCMakeTest.cmake b/Tests/RunCMake/Syntax/RunCMakeTest.cmake index b8f5fd0..a397620 100644 --- a/Tests/RunCMake/Syntax/RunCMakeTest.cmake +++ b/Tests/RunCMake/Syntax/RunCMakeTest.cmake @@ -112,6 +112,9 @@ run_cmake(CMP0053-NameWithCarriageReturnQuoted) run_cmake(CMP0053-NameWithEscapedSpacesQuoted) run_cmake(CMP0053-NameWithEscapedTabsQuoted) +# Variable special types +run_cmake(QueryCache) + # Function and macro tests. run_cmake(FunctionUnmatched) run_cmake(FunctionUnmatchedForeach) diff --git a/Tests/RunCMake/export/ExportPropertiesUndefined.cmake b/Tests/RunCMake/export/ExportPropertiesUndefined.cmake new file mode 100644 index 0000000..aa529f2 --- /dev/null +++ b/Tests/RunCMake/export/ExportPropertiesUndefined.cmake @@ -0,0 +1,11 @@ +enable_language(CXX) +add_library(foo empty.cpp) +set_target_properties(foo PROPERTIES + EXPORT_PROPERTIES "NotDefinedProp" +) +export(TARGETS foo FILE "${CMAKE_CURRENT_BINARY_DIR}/foo.cmake") +install(TARGETS foo EXPORT fooExport + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) diff --git a/Tests/RunCMake/export/RunCMakeTest.cmake b/Tests/RunCMake/export/RunCMakeTest.cmake index 10ced90..46bb1fc 100644 --- a/Tests/RunCMake/export/RunCMakeTest.cmake +++ b/Tests/RunCMake/export/RunCMakeTest.cmake @@ -8,3 +8,4 @@ run_cmake(NoExportSet) run_cmake(ForbiddenToExportInterfaceProperties) run_cmake(ForbiddenToExportImportedProperties) run_cmake(ForbiddenToExportPropertyWithGenExp) +run_cmake(ExportPropertiesUndefined) diff --git a/Tests/RunCMake/target_link_options/LINK_OPTIONS-interface-static-check.cmake b/Tests/RunCMake/target_link_options/LINK_OPTIONS-interface-static-check.cmake new file mode 100644 index 0000000..6c5ffdb --- /dev/null +++ b/Tests/RunCMake/target_link_options/LINK_OPTIONS-interface-static-check.cmake @@ -0,0 +1,4 @@ + +if (NOT actual_stdout MATCHES "BADFLAG_INTERFACE") + set (RunCMake_TEST_FAILED "Not found expected 'BADFLAG_INTERFACE'.") +endif() diff --git a/Tests/RunCMake/target_link_options/LINK_OPTIONS-interface-static-result.txt b/Tests/RunCMake/target_link_options/LINK_OPTIONS-interface-static-result.txt new file mode 100644 index 0000000..8d98f9d --- /dev/null +++ b/Tests/RunCMake/target_link_options/LINK_OPTIONS-interface-static-result.txt @@ -0,0 +1 @@ +.* diff --git a/Tests/RunCMake/target_link_options/LINK_OPTIONS-static-check.cmake b/Tests/RunCMake/target_link_options/LINK_OPTIONS-static-check.cmake new file mode 100644 index 0000000..b68e451 --- /dev/null +++ b/Tests/RunCMake/target_link_options/LINK_OPTIONS-static-check.cmake @@ -0,0 +1,7 @@ + +if (actual_stdout MATCHES "BADFLAG_RELEASE") + set (RunCMake_TEST_FAILED "Found 'BADFLAG_RELEASE' which was not expected.") +endif() +if (actual_stdout MATCHES "SHELL:") + string (APPEND RunCMake_TEST_FAILED "\nFound unexpected prefix 'SHELL:'.") +endif() diff --git a/Tests/RunCMake/target_link_options/LINK_OPTIONS-static-result.txt b/Tests/RunCMake/target_link_options/LINK_OPTIONS-static-result.txt new file mode 100644 index 0000000..573541a --- /dev/null +++ b/Tests/RunCMake/target_link_options/LINK_OPTIONS-static-result.txt @@ -0,0 +1 @@ +0 diff --git a/Tests/RunCMake/target_link_options/LINK_OPTIONS.cmake b/Tests/RunCMake/target_link_options/LINK_OPTIONS.cmake index f17195d..bb04841 100644 --- a/Tests/RunCMake/target_link_options/LINK_OPTIONS.cmake +++ b/Tests/RunCMake/target_link_options/LINK_OPTIONS.cmake @@ -22,6 +22,22 @@ add_executable(LinkOptions_consumer LinkOptionsExe.c) target_link_libraries(LinkOptions_consumer PRIVATE LinkOptions_producer) +# static library with INTERFACE_LINK_OPTIONS +add_library(LinkOptions_producer_static STATIC LinkOptionsLib.c) +target_link_options(LinkOptions_producer_static + INTERFACE ${pre}BADFLAG_INTERFACE${obj}) + +add_executable(LinkOptions_consumer_static LinkOptionsExe.c) +target_link_libraries(LinkOptions_consumer_static PRIVATE LinkOptions_producer_static) + + +# static library with generator expression +add_library(LinkOptions_static STATIC LinkOptionsLib.c) +target_link_options(LinkOptions_static PRIVATE $<$<CONFIG:Release>:${pre}BADFLAG_RELEASE${obj}> + "SHELL:" # produces no options + ) + + # shared library with generator expression add_library(LinkOptions_shared SHARED LinkOptionsLib.c) target_link_options(LinkOptions_shared PRIVATE $<$<CONFIG:Release>:${pre}BADFLAG_RELEASE${obj}> diff --git a/Tests/RunCMake/target_link_options/RunCMakeTest.cmake b/Tests/RunCMake/target_link_options/RunCMakeTest.cmake index 121a0ce..1eaa5d2 100644 --- a/Tests/RunCMake/target_link_options/RunCMakeTest.cmake +++ b/Tests/RunCMake/target_link_options/RunCMakeTest.cmake @@ -21,6 +21,8 @@ if (NOT CMAKE_C_COMPILER_ID STREQUAL "Intel") run_cmake_target(LINK_OPTIONS basic LinkOptions) run_cmake_target(LINK_OPTIONS interface LinkOptions_consumer) + run_cmake_target(LINK_OPTIONS interface-static LinkOptions_consumer_static) + run_cmake_target(LINK_OPTIONS static LinkOptions_static --config Release) run_cmake_target(LINK_OPTIONS shared LinkOptions_shared --config Release) run_cmake_target(LINK_OPTIONS mod LinkOptions_mod --config Release) run_cmake_target(LINK_OPTIONS exe LinkOptions_exe --config Release) diff --git a/Utilities/IWYU/mapping.imp b/Utilities/IWYU/mapping.imp index 77bb572..5af8b85 100644 --- a/Utilities/IWYU/mapping.imp +++ b/Utilities/IWYU/mapping.imp @@ -53,7 +53,7 @@ #{ symbol: [ "std::pair", private, "<set>", public ] }, # HACK: iwyu wrongly thinks that <system_error> is needed for std::hash - { symbol: [ "std::hash", private, "<utility>", public ] }, + { symbol: [ "std::hash", private, "<functional>", public ] }, # __decay_and_strip is used internally in the C++11 standard library. # IWYU does not classify it as internal and suggests to add <type_traits>. diff --git a/Utilities/Scripts/update-liblzma.bash b/Utilities/Scripts/update-liblzma.bash index 088eb91..fdf66b3 100755 --- a/Utilities/Scripts/update-liblzma.bash +++ b/Utilities/Scripts/update-liblzma.bash @@ -7,8 +7,8 @@ shopt -s dotglob readonly name="liblzma" readonly ownership="liblzma upstream <xz-devel@tukaani.org>" readonly subtree="Utilities/cmliblzma" -readonly repo="http://git.tukaani.org/xz.git" -readonly tag="v5.0.8" +readonly repo="https://git.tukaani.org/xz.git" +readonly tag="v5.2.4" readonly shortlog=false readonly paths=" COPYING @@ -24,6 +24,10 @@ extract_source () { mv src/common . mv src/liblzma . rmdir src + rm liblzma/Makefile.* + rm liblzma/*/Makefile.* + rm liblzma/liblzma.map + rm liblzma/validate_map.sh popd } diff --git a/Utilities/cmliblzma/CMakeLists.txt b/Utilities/cmliblzma/CMakeLists.txt index bb3b8a7..e9f8826 100644 --- a/Utilities/cmliblzma/CMakeLists.txt +++ b/Utilities/cmliblzma/CMakeLists.txt @@ -6,22 +6,6 @@ include(CheckSymbolExists) include(CheckTypeSize) include(TestBigEndian) -CHECK_C_SOURCE_COMPILES( - "int test (void *restrict x);\nint main (void) {return 0;}" - HAVE_RESTRICT) - -CHECK_C_SOURCE_COMPILES( -"typedef struct abc *d;\nint test (d __restrict x);\nint main (void) {return 0;}" - HAVE___RESTRICT) - -CHECK_C_SOURCE_COMPILES( - "static inline int test (void) {return 0;}\nint main (void) {return test();}" - HAVE_INLINE) - -CHECK_C_SOURCE_COMPILES ( - "static __inline int test (void) {return 0;}\nint main (void) {return test();}" - HAVE___INLINE) - CHECK_INCLUDE_FILE(byteswap.h HAVE_BYTESWAP_H) CHECK_INCLUDE_FILE(inttypes.h HAVE_INTTYPES_H) CHECK_INCLUDE_FILE(limits.h HAVE_LIMITS_H) @@ -95,7 +79,7 @@ CHECK_TYPE_SIZE("unsigned short" SIZE_OF_UNSIGNED_SHORT) CHECK_TYPE_SIZE("unsigned" SIZE_OF_UNSIGNED) CHECK_TYPE_SIZE("unsigned long" SIZE_OF_UNSIGNED_LONG) CHECK_TYPE_SIZE("unsigned long long" SIZE_OF_UNSIGNED_LONG_LONG) -CHECK_TYPE_SIZE("size_t" SIZE_OF_SIZE_T) +CHECK_TYPE_SIZE("size_t" SIZEOF_SIZE_T) CHECK_TYPE_SIZE("__int64" __INT64) CHECK_TYPE_SIZE("unsigned __int64" UNSIGNED___INT64) diff --git a/Utilities/cmliblzma/COPYING b/Utilities/cmliblzma/COPYING index 43c90d0..20e60d5 100644 --- a/Utilities/cmliblzma/COPYING +++ b/Utilities/cmliblzma/COPYING @@ -47,7 +47,7 @@ XZ Utils Licensing naturally it is not legally required. Here is an example of a good notice to put into "about box" or into documentation: - This software includes code from XZ Utils <http://tukaani.org/xz/>. + This software includes code from XZ Utils <https://tukaani.org/xz/>. The following license texts are included in the following files: - COPYING.LGPLv2.1: GNU Lesser General Public License version 2.1 diff --git a/Utilities/cmliblzma/common/common_w32res.rc b/Utilities/cmliblzma/common/common_w32res.rc index fdb88d1..a70de34 100644 --- a/Utilities/cmliblzma/common/common_w32res.rc +++ b/Utilities/cmliblzma/common/common_w32res.rc @@ -17,7 +17,7 @@ #define MY_VERSION LZMA_VERSION_MAJOR,LZMA_VERSION_MINOR,LZMA_VERSION_PATCH,MY_BUILD #define MY_FILENAME MY_NAME MY_SUFFIX -#define MY_COMPANY "The Tukaani Project <http://tukaani.org/>" +#define MY_COMPANY "The Tukaani Project <https://tukaani.org/>" #define MY_PRODUCT PACKAGE_NAME " <" PACKAGE_URL ">" LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US diff --git a/Utilities/cmliblzma/common/sysdefs.h b/Utilities/cmliblzma/common/sysdefs.h index a6edea8..22f487b 100644 --- a/Utilities/cmliblzma/common/sysdefs.h +++ b/Utilities/cmliblzma/common/sysdefs.h @@ -18,6 +18,7 @@ #if defined(_MSC_VER) # pragma warning(push,1) +# pragma warning(disable: 4028) /* formal parameter different from decl */ # pragma warning(disable: 4142) /* benign redefinition of type */ # pragma warning(disable: 4761) /* integral size mismatch in argument */ #endif @@ -124,9 +125,9 @@ // The code currently assumes that size_t is either 32-bit or 64-bit. #ifndef SIZE_MAX -# if SIZE_OF_SIZE_T == 4 +# if SIZEOF_SIZE_T == 4 # define SIZE_MAX UINT32_MAX -# elif SIZE_OF_SIZE_T == 8 +# elif SIZEOF_SIZE_T == 8 # define SIZE_MAX UINT64_MAX # else # error size_t is not 32-bit or 64-bit @@ -175,6 +176,16 @@ typedef unsigned char _Bool; # include <memory.h> #endif +// As of MSVC 2013, inline and restrict are supported with +// non-standard keywords. +#if defined(_WIN32) && defined(_MSC_VER) +# ifndef inline +# define inline __inline +# endif +# ifndef restrict +# define restrict __restrict +# endif +#endif //////////// // Macros // diff --git a/Utilities/cmliblzma/common/tuklib_integer.h b/Utilities/cmliblzma/common/tuklib_integer.h index 5e8262a..5d82685 100644 --- a/Utilities/cmliblzma/common/tuklib_integer.h +++ b/Utilities/cmliblzma/common/tuklib_integer.h @@ -106,6 +106,17 @@ #endif +//////////////////////////////// +// Compiler-specific features // +//////////////////////////////// + +// Newer Intel C compilers require immintrin.h for _bit_scan_reverse() +// and such functions. +#if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1500) +# include <immintrin.h> +#endif + + /////////////////// // Byte swapping // /////////////////// @@ -329,8 +340,8 @@ unaligned_read32le(const uint8_t *buf) static inline void unaligned_write16be(uint8_t *buf, uint16_t num) { - buf[0] = num >> 8; - buf[1] = num; + buf[0] = (uint8_t)(num >> 8); + buf[1] = (uint8_t)num; return; } @@ -338,8 +349,8 @@ unaligned_write16be(uint8_t *buf, uint16_t num) static inline void unaligned_write16le(uint8_t *buf, uint16_t num) { - buf[0] = num; - buf[1] = num >> 8; + buf[0] = (uint8_t)num; + buf[1] = (uint8_t)(num >> 8); return; } @@ -347,10 +358,10 @@ unaligned_write16le(uint8_t *buf, uint16_t num) static inline void unaligned_write32be(uint8_t *buf, uint32_t num) { - buf[0] = num >> 24; - buf[1] = num >> 16; - buf[2] = num >> 8; - buf[3] = num; + buf[0] = (uint8_t)(num >> 24); + buf[1] = (uint8_t)(num >> 16); + buf[2] = (uint8_t)(num >> 8); + buf[3] = (uint8_t)num; return; } @@ -358,10 +369,10 @@ unaligned_write32be(uint8_t *buf, uint32_t num) static inline void unaligned_write32le(uint8_t *buf, uint32_t num) { - buf[0] = num; - buf[1] = num >> 8; - buf[2] = num >> 16; - buf[3] = num >> 24; + buf[0] = (uint8_t)num; + buf[1] = (uint8_t)(num >> 8); + buf[2] = (uint8_t)(num >> 16); + buf[3] = (uint8_t)(num >> 24); return; } diff --git a/Utilities/cmliblzma/config.h.in b/Utilities/cmliblzma/config.h.in index 9c53150..06f7fcb 100644 --- a/Utilities/cmliblzma/config.h.in +++ b/Utilities/cmliblzma/config.h.in @@ -29,7 +29,7 @@ @SIZE_OF_UNSIGNED_CODE@ @SIZE_OF_UNSIGNED_LONG_CODE@ @SIZE_OF_UNSIGNED_LONG_LONG_CODE@ -@SIZE_OF_SIZE_T_CODE@ +@SIZEOF_SIZE_T_CODE@ /* * If we lack int64_t, define it to the first of __int64, int, long, and long long @@ -180,32 +180,6 @@ typedef uint64_t uintmax_t; #cmakedefine uintptr_t @uintptr_t@ - -#cmakedefine HAVE_RESTRICT -#cmakedefine HAVE___RESTRICT - -#cmakedefine HAVE_INLINE -#cmakedefine HAVE___INLINE - -#ifndef HAVE_RESTRICT -# ifdef HAVE___RESTRICT -# define LZMA_RESTRICT __restrict -# else -# define LZMA_RESTRICT -# endif -#else -# define LZMA_RESTRICT restrict -#endif /* HAVE_RESTRICT */ - -#ifndef HAVE_INLINE -# ifdef HAVE___INLINE -# define inline __inline -# else -# define inline -# endif -#endif /* HAVE_INLINE */ - - #cmakedefine WORDS_BIGENDIAN 1 #cmakedefine HAVE_BYTESWAP_H 1 diff --git a/Utilities/cmliblzma/liblzma/api/lzma.h b/Utilities/cmliblzma/liblzma/api/lzma.h index fb874c3..aa88e42 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma.h +++ b/Utilities/cmliblzma/liblzma/api/lzma.h @@ -82,12 +82,20 @@ # if !defined(UINT32_C) || !defined(UINT64_C) \ || !defined(UINT32_MAX) || !defined(UINT64_MAX) /* - * MSVC has no C99 support, and thus it cannot be used to - * compile liblzma. The liblzma API has to still be usable - * from MSVC, so we need to define the required standard - * integer types here. + * MSVC versions older than 2013 have no C99 support, and + * thus they cannot be used to compile liblzma. Using an + * existing liblzma.dll with old MSVC can work though(*), + * but we need to define the required standard integer + * types here in a MSVC-specific way. + * + * (*) If you do this, the existing liblzma.dll probably uses + * a different runtime library than your MSVC-built + * application. Mixing runtimes is generally bad, but + * in this case it should work as long as you avoid + * the few rarely-needed liblzma functions that allocate + * memory and expect the caller to free it using free(). */ -# if defined(_WIN32) && defined(_MSC_VER) +# if defined(_WIN32) && defined(_MSC_VER) && _MSC_VER < 1800 typedef unsigned __int8 uint8_t; typedef unsigned __int32 uint32_t; typedef unsigned __int64 uint64_t; @@ -211,7 +219,11 @@ */ #ifndef lzma_nothrow # if defined(__cplusplus) -# define lzma_nothrow throw() +# if __cplusplus >= 201103L +# define lzma_nothrow noexcept +# else +# define lzma_nothrow throw() +# endif # elif __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 3) # define lzma_nothrow __attribute__((__nothrow__)) # else @@ -286,7 +298,7 @@ extern "C" { #include "lzma/filter.h" #include "lzma/bcj.h" #include "lzma/delta.h" -#include "lzma/lzma.h" +#include "lzma/lzma12.h" /* Container formats */ #include "lzma/container.h" diff --git a/Utilities/cmliblzma/liblzma/api/lzma/base.h b/Utilities/cmliblzma/liblzma/api/lzma/base.h index 43dde8d..a6005ac 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/base.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/base.h @@ -240,12 +240,12 @@ typedef enum { /** * \brief The `action' argument for lzma_code() * - * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, or LZMA_FINISH, - * the same `action' must is used until lzma_code() returns LZMA_STREAM_END. - * Also, the amount of input (that is, strm->avail_in) must not be modified - * by the application until lzma_code() returns LZMA_STREAM_END. Changing the - * `action' or modifying the amount of input will make lzma_code() return - * LZMA_PROG_ERROR. + * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, LZMA_FULL_BARRIER, + * or LZMA_FINISH, the same `action' must is used until lzma_code() returns + * LZMA_STREAM_END. Also, the amount of input (that is, strm->avail_in) must + * not be modified by the application until lzma_code() returns + * LZMA_STREAM_END. Changing the `action' or modifying the amount of input + * will make lzma_code() return LZMA_PROG_ERROR. */ typedef enum { LZMA_RUN = 0, @@ -293,7 +293,7 @@ typedef enum { * * All the input data going to the current Block must have * been given to the encoder (the last bytes can still be - * pending in* next_in). Call lzma_code() with LZMA_FULL_FLUSH + * pending in *next_in). Call lzma_code() with LZMA_FULL_FLUSH * until it returns LZMA_STREAM_END. Then continue normally * with LZMA_RUN or finish the Stream with LZMA_FINISH. * @@ -302,6 +302,29 @@ typedef enum { * no unfinished Block, no empty Block is created. */ + LZMA_FULL_BARRIER = 4, + /**< + * \brief Finish encoding of the current Block + * + * This is like LZMA_FULL_FLUSH except that this doesn't + * necessarily wait until all the input has been made + * available via the output buffer. That is, lzma_code() + * might return LZMA_STREAM_END as soon as all the input + * has been consumed (avail_in == 0). + * + * LZMA_FULL_BARRIER is useful with a threaded encoder if + * one wants to split the .xz Stream into Blocks at specific + * offsets but doesn't care if the output isn't flushed + * immediately. Using LZMA_FULL_BARRIER allows keeping + * the threads busy while LZMA_FULL_FLUSH would make + * lzma_code() wait until all the threads have finished + * until more data could be passed to the encoder. + * + * With a lzma_stream initialized with the single-threaded + * lzma_stream_encoder() or lzma_easy_encoder(), + * LZMA_FULL_BARRIER is an alias for LZMA_FULL_FLUSH. + */ + LZMA_FINISH = 3 /**< * \brief Finish the coding operation @@ -332,11 +355,19 @@ typedef enum { * malloc() and free(). C++ users should note that the custom memory * handling functions must not throw exceptions. * - * liblzma doesn't make an internal copy of lzma_allocator. Thus, it is - * OK to change these function pointers in the middle of the coding - * process, but obviously it must be done carefully to make sure that the - * replacement `free' can deallocate memory allocated by the earlier - * `alloc' function(s). + * Single-threaded mode only: liblzma doesn't make an internal copy of + * lzma_allocator. Thus, it is OK to change these function pointers in + * the middle of the coding process, but obviously it must be done + * carefully to make sure that the replacement `free' can deallocate + * memory allocated by the earlier `alloc' function(s). + * + * Multithreaded mode: liblzma might internally store pointers to the + * lzma_allocator given via the lzma_stream structure. The application + * must not change the allocator pointer in lzma_stream or the contents + * of the pointed lzma_allocator structure until lzma_end() has been used + * to free the memory associated with that lzma_stream. The allocation + * functions might be called simultaneously from multiple threads, and + * thus they must be thread safe. */ typedef struct { /** @@ -448,7 +479,8 @@ typedef struct lzma_internal_s lzma_internal; * * Application may modify the values of total_in and total_out as it wants. * They are updated by liblzma to match the amount of data read and - * written, but aren't used for anything else. + * written but aren't used for anything else except as a possible return + * values from lzma_get_progress(). */ typedef struct { const uint8_t *next_in; /**< Pointer to the next input byte. */ @@ -464,8 +496,10 @@ typedef struct { * * In most cases this is NULL which makes liblzma use * the standard malloc() and free(). + * + * \note In 5.0.x this is not a const pointer. */ - lzma_allocator *allocator; + const lzma_allocator *allocator; /** Internal state is not visible to applications. */ lzma_internal *internal; @@ -547,6 +581,25 @@ extern LZMA_API(void) lzma_end(lzma_stream *strm) lzma_nothrow; /** + * \brief Get progress information + * + * In single-threaded mode, applications can get progress information from + * strm->total_in and strm->total_out. In multi-threaded mode this is less + * useful because a significant amount of both input and output data gets + * buffered internally by liblzma. This makes total_in and total_out give + * misleading information and also makes the progress indicator updates + * non-smooth. + * + * This function gives realistic progress information also in multi-threaded + * mode by taking into account the progress made by each thread. In + * single-threaded mode *progress_in and *progress_out are set to + * strm->total_in and strm->total_out, respectively. + */ +extern LZMA_API(void) lzma_get_progress(lzma_stream *strm, + uint64_t *progress_in, uint64_t *progress_out) lzma_nothrow; + + +/** * \brief Get the memory usage of decoder filter chain * * This function is currently supported only when *strm has been initialized @@ -591,11 +644,16 @@ extern LZMA_API(uint64_t) lzma_memlimit_get(const lzma_stream *strm) * This function is supported only when *strm has been initialized with * a function that takes a memlimit argument. * + * liblzma 5.2.3 and earlier has a bug where memlimit value of 0 causes + * this function to do nothing (leaving the limit unchanged) and still + * return LZMA_OK. Later versions treat 0 as if 1 had been specified (so + * lzma_memlimit_get() will return 1 even if you specify 0 here). + * * \return - LZMA_OK: New memory usage limit successfully set. * - LZMA_MEMLIMIT_ERROR: The new limit is too small. * The limit was not changed. * - LZMA_PROG_ERROR: Invalid arguments, e.g. *strm doesn't - * support memory usage limit or memlimit was zero. + * support memory usage limit. */ extern LZMA_API(lzma_ret) lzma_memlimit_set( lzma_stream *strm, uint64_t memlimit) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/block.h b/Utilities/cmliblzma/liblzma/api/lzma/block.h index e6710a7..7bdcfd7 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/block.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/block.h @@ -31,11 +31,16 @@ typedef struct { /** * \brief Block format version * - * To prevent API and ABI breakages if new features are needed in - * the Block field, a version number is used to indicate which - * fields in this structure are in use. For now, version must always - * be zero. With non-zero version, most Block related functions will - * return LZMA_OPTIONS_ERROR. + * To prevent API and ABI breakages when new features are needed, + * a version number is used to indicate which fields in this + * structure are in use: + * - liblzma >= 5.0.0: version = 0 is supported. + * - liblzma >= 5.1.4beta: Support for version = 1 was added, + * which adds the ignore_check field. + * + * If version is greater than one, most Block related functions + * will return LZMA_OPTIONS_ERROR (lzma_block_header_decode() works + * with any version value). * * Read by: * - All functions that take pointer to lzma_block as argument, @@ -233,7 +238,28 @@ typedef struct { lzma_reserved_enum reserved_enum2; lzma_reserved_enum reserved_enum3; lzma_reserved_enum reserved_enum4; - lzma_bool reserved_bool1; + + /** + * \brief A flag to Block decoder to not verify the Check field + * + * This field is supported by liblzma >= 5.1.4beta if .version >= 1. + * + * If this is set to true, the integrity check won't be calculated + * and verified. Unless you know what you are doing, you should + * leave this to false. (A reason to set this to true is when the + * file integrity is verified externally anyway and you want to + * speed up the decompression, which matters mostly when using + * SHA-256 as the integrity check.) + * + * If .version >= 1, read by: + * - lzma_block_decoder() + * - lzma_block_buffer_decode() + * + * Written by (.version is ignored): + * - lzma_block_header_decode() always sets this to false + */ + lzma_bool ignore_check; + lzma_bool reserved_bool2; lzma_bool reserved_bool3; lzma_bool reserved_bool4; @@ -310,10 +336,14 @@ extern LZMA_API(lzma_ret) lzma_block_header_encode( /** * \brief Decode Block Header * - * block->version should be set to the highest value supported by the - * application; currently the only possible version is zero. This function - * will set version to the lowest value that still supports all the features - * required by the Block Header. + * block->version should (usually) be set to the highest value supported + * by the application. If the application sets block->version to a value + * higher than supported by the current liblzma version, this function will + * downgrade block->version to the highest value supported by it. Thus one + * should check the value of block->version after calling this function if + * block->version was set to a non-zero value and the application doesn't + * otherwise know that the liblzma version being used is new enough to + * support the specified block->version. * * The size of the Block Header must have already been decoded with * lzma_block_header_size_decode() macro and stored to block->header_size. @@ -344,7 +374,7 @@ extern LZMA_API(lzma_ret) lzma_block_header_encode( * block->header_size is invalid or block->filters is NULL. */ extern LZMA_API(lzma_ret) lzma_block_header_decode(lzma_block *block, - lzma_allocator *allocator, const uint8_t *in) + const lzma_allocator *allocator, const uint8_t *in) lzma_nothrow lzma_attr_warn_unused_result; @@ -493,7 +523,25 @@ extern LZMA_API(size_t) lzma_block_buffer_bound(size_t uncompressed_size) * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_block_buffer_encode( - lzma_block *block, lzma_allocator *allocator, + lzma_block *block, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** + * \brief Single-call uncompressed .xz Block encoder + * + * This is like lzma_block_buffer_encode() except this doesn't try to + * compress the data and instead encodes the data using LZMA2 uncompressed + * chunks. The required output buffer size can be determined with + * lzma_block_buffer_bound(). + * + * Since the data won't be compressed, this function ignores block->filters. + * This function doesn't take lzma_allocator because this function doesn't + * allocate any memory from the heap. + */ +extern LZMA_API(lzma_ret) lzma_block_uncomp_encode(lzma_block *block, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -527,7 +575,7 @@ extern LZMA_API(lzma_ret) lzma_block_buffer_encode( * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_block_buffer_decode( - lzma_block *block, lzma_allocator *allocator, + lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/container.h b/Utilities/cmliblzma/liblzma/api/lzma/container.h index 7a9ffc6..9fbf4df 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/container.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/container.h @@ -61,6 +61,131 @@ /** + * \brief Multithreading options + */ +typedef struct { + /** + * \brief Flags + * + * Set this to zero if no flags are wanted. + * + * No flags are currently supported. + */ + uint32_t flags; + + /** + * \brief Number of worker threads to use + */ + uint32_t threads; + + /** + * \brief Maximum uncompressed size of a Block + * + * The encoder will start a new .xz Block every block_size bytes. + * Using LZMA_FULL_FLUSH or LZMA_FULL_BARRIER with lzma_code() + * the caller may tell liblzma to start a new Block earlier. + * + * With LZMA2, a recommended block size is 2-4 times the LZMA2 + * dictionary size. With very small dictionaries, it is recommended + * to use at least 1 MiB block size for good compression ratio, even + * if this is more than four times the dictionary size. Note that + * these are only recommendations for typical use cases; feel free + * to use other values. Just keep in mind that using a block size + * less than the LZMA2 dictionary size is waste of RAM. + * + * Set this to 0 to let liblzma choose the block size depending + * on the compression options. For LZMA2 it will be 3*dict_size + * or 1 MiB, whichever is more. + * + * For each thread, about 3 * block_size bytes of memory will be + * allocated. This may change in later liblzma versions. If so, + * the memory usage will probably be reduced, not increased. + */ + uint64_t block_size; + + /** + * \brief Timeout to allow lzma_code() to return early + * + * Multithreading can make liblzma to consume input and produce + * output in a very bursty way: it may first read a lot of input + * to fill internal buffers, then no input or output occurs for + * a while. + * + * In single-threaded mode, lzma_code() won't return until it has + * either consumed all the input or filled the output buffer. If + * this is done in multithreaded mode, it may cause a call + * lzma_code() to take even tens of seconds, which isn't acceptable + * in all applications. + * + * To avoid very long blocking times in lzma_code(), a timeout + * (in milliseconds) may be set here. If lzma_code() would block + * longer than this number of milliseconds, it will return with + * LZMA_OK. Reasonable values are 100 ms or more. The xz command + * line tool uses 300 ms. + * + * If long blocking times are fine for you, set timeout to a special + * value of 0, which will disable the timeout mechanism and will make + * lzma_code() block until all the input is consumed or the output + * buffer has been filled. + * + * \note Even with a timeout, lzma_code() might sometimes take + * somewhat long time to return. No timing guarantees + * are made. + */ + uint32_t timeout; + + /** + * \brief Compression preset (level and possible flags) + * + * The preset is set just like with lzma_easy_encoder(). + * The preset is ignored if filters below is non-NULL. + */ + uint32_t preset; + + /** + * \brief Filter chain (alternative to a preset) + * + * If this is NULL, the preset above is used. Otherwise the preset + * is ignored and the filter chain specified here is used. + */ + const lzma_filter *filters; + + /** + * \brief Integrity check type + * + * See check.h for available checks. The xz command line tool + * defaults to LZMA_CHECK_CRC64, which is a good choice if you + * are unsure. + */ + lzma_check check; + + /* + * Reserved space to allow possible future extensions without + * breaking the ABI. You should not touch these, because the names + * of these variables may change. These are and will never be used + * with the currently supported options, so it is safe to leave these + * uninitialized. + */ + lzma_reserved_enum reserved_enum1; + lzma_reserved_enum reserved_enum2; + lzma_reserved_enum reserved_enum3; + uint32_t reserved_int1; + uint32_t reserved_int2; + uint32_t reserved_int3; + uint32_t reserved_int4; + uint64_t reserved_int5; + uint64_t reserved_int6; + uint64_t reserved_int7; + uint64_t reserved_int8; + void *reserved_ptr1; + void *reserved_ptr2; + void *reserved_ptr3; + void *reserved_ptr4; + +} lzma_mt; + + +/** * \brief Calculate approximate memory usage of easy encoder * * This function is a wrapper for lzma_raw_encoder_memusage(). @@ -165,7 +290,8 @@ extern LZMA_API(lzma_ret) lzma_easy_encoder( */ extern LZMA_API(lzma_ret) lzma_easy_buffer_encode( uint32_t preset, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -191,6 +317,49 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder(lzma_stream *strm, /** + * \brief Calculate approximate memory usage of multithreaded .xz encoder + * + * Since doing the encoding in threaded mode doesn't affect the memory + * requirements of single-threaded decompressor, you can use + * lzma_easy_decoder_memusage(options->preset) or + * lzma_raw_decoder_memusage(options->filters) to calculate + * the decompressor memory requirements. + * + * \param options Compression options + * + * \return Number of bytes of memory required for encoding with the + * given options. If an error occurs, for example due to + * unsupported preset or filter chain, UINT64_MAX is returned. + */ +extern LZMA_API(uint64_t) lzma_stream_encoder_mt_memusage( + const lzma_mt *options) lzma_nothrow lzma_attr_pure; + + +/** + * \brief Initialize multithreaded .xz Stream encoder + * + * This provides the functionality of lzma_easy_encoder() and + * lzma_stream_encoder() as a single function for multithreaded use. + * + * The supported actions for lzma_code() are LZMA_RUN, LZMA_FULL_FLUSH, + * LZMA_FULL_BARRIER, and LZMA_FINISH. Support for LZMA_SYNC_FLUSH might be + * added in the future. + * + * \param strm Pointer to properly prepared lzma_stream + * \param options Pointer to multithreaded compression options + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_UNSUPPORTED_CHECK + * - LZMA_OPTIONS_ERROR + * - LZMA_PROG_ERROR + */ +extern LZMA_API(lzma_ret) lzma_stream_encoder_mt( + lzma_stream *strm, const lzma_mt *options) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** * \brief Initialize .lzma encoder (legacy file format) * * The .lzma format is sometimes called the LZMA_Alone format, which is the @@ -269,7 +438,8 @@ extern LZMA_API(size_t) lzma_stream_buffer_bound(size_t uncompressed_size) */ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( lzma_filter *filters, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -305,6 +475,30 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( /** + * This flag makes lzma_code() not calculate and verify the integrity check + * of the compressed data in .xz files. This means that invalid integrity + * check values won't be detected and LZMA_DATA_ERROR won't be returned in + * such cases. + * + * This flag only affects the checks of the compressed data itself; the CRC32 + * values in the .xz headers will still be verified normally. + * + * Don't use this flag unless you know what you are doing. Possible reasons + * to use this flag: + * + * - Trying to recover data from a corrupt .xz file. + * + * - Speeding up decompression, which matters mostly with SHA-256 + * or with files that have compressed extremely well. It's recommended + * to not use this flag for this purpose unless the file integrity is + * verified externally in some other way. + * + * Support for this flag was added in liblzma 5.1.4beta. + */ +#define LZMA_IGNORE_CHECK UINT32_C(0x10) + + +/** * This flag enables decoding of concatenated files with file formats that * allow concatenating compressed files as is. From the formats currently * supported by liblzma, only the .xz format allows concatenated files. @@ -326,7 +520,10 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( * * \param strm Pointer to properly prepared lzma_stream * \param memlimit Memory usage limit as bytes. Use UINT64_MAX - * to effectively disable the limiter. + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. * \param flags Bitwise-or of zero or more of the decoder flags: * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, * LZMA_TELL_ANY_CHECK, LZMA_CONCATENATED @@ -350,7 +547,10 @@ extern LZMA_API(lzma_ret) lzma_stream_decoder( * * \param strm Pointer to properly prepared lzma_stream * \param memlimit Memory usage limit as bytes. Use UINT64_MAX - * to effectively disable the limiter. + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. * \param flags Bitwise-or of flags, or zero for no flags. * * \return - LZMA_OK: Initialization was successful. @@ -366,9 +566,16 @@ extern LZMA_API(lzma_ret) lzma_auto_decoder( /** * \brief Initialize .lzma decoder (legacy file format) * + * \param strm Pointer to properly prepared lzma_stream + * \param memlimit Memory usage limit as bytes. Use UINT64_MAX + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. + * * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. - * There is no need to use LZMA_FINISH, but allowing it may simplify - * certain types of applications. + * There is no need to use LZMA_FINISH, but it's allowed because it may + * simplify certain types of applications. * * \return - LZMA_OK * - LZMA_MEM_ERROR @@ -418,7 +625,8 @@ extern LZMA_API(lzma_ret) lzma_alone_decoder( * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_buffer_decode( - uint64_t *memlimit, uint32_t flags, lzma_allocator *allocator, + uint64_t *memlimit, uint32_t flags, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/filter.h b/Utilities/cmliblzma/liblzma/api/lzma/filter.h index e0bc163..4e78752 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/filter.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/filter.h @@ -116,8 +116,9 @@ extern LZMA_API(lzma_bool) lzma_filter_decoder_is_supported(lzma_vli id) * is not NULL. * - LZMA_PROG_ERROR: src or dest is NULL. */ -extern LZMA_API(lzma_ret) lzma_filters_copy(const lzma_filter *src, - lzma_filter *dest, lzma_allocator *allocator) lzma_nothrow; +extern LZMA_API(lzma_ret) lzma_filters_copy( + const lzma_filter *src, lzma_filter *dest, + const lzma_allocator *allocator) lzma_nothrow; /** @@ -256,7 +257,7 @@ extern LZMA_API(lzma_ret) lzma_filters_update( * won't necessarily meet that bound.) */ extern LZMA_API(lzma_ret) lzma_raw_buffer_encode( - const lzma_filter *filters, lzma_allocator *allocator, + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -280,7 +281,7 @@ extern LZMA_API(lzma_ret) lzma_raw_buffer_encode( * which no data is written to is out[out_size]. */ extern LZMA_API(lzma_ret) lzma_raw_buffer_decode( - const lzma_filter *filters, lzma_allocator *allocator, + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -356,7 +357,7 @@ extern LZMA_API(lzma_ret) lzma_properties_encode( * - LZMA_MEM_ERROR */ extern LZMA_API(lzma_ret) lzma_properties_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) lzma_nothrow; @@ -419,6 +420,6 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filter, * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_filter_flags_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) lzma_nothrow lzma_attr_warn_unused_result; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/hardware.h b/Utilities/cmliblzma/liblzma/api/lzma/hardware.h index e7dd03c..5321d9a 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/hardware.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/hardware.h @@ -48,3 +48,17 @@ * of RAM on the specific operating system. */ extern LZMA_API(uint64_t) lzma_physmem(void) lzma_nothrow; + + +/** + * \brief Get the number of processor cores or threads + * + * This function may be useful when determining how many threads to use. + * If the hardware supports more than one thread per CPU core, the number + * of hardware threads is returned if that information is available. + * + * \brief On success, the number of available CPU threads or cores is + * returned. If this information isn't available or an error + * occurs, zero is returned. + */ +extern LZMA_API(uint32_t) lzma_cputhreads(void) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/index.h b/Utilities/cmliblzma/liblzma/api/lzma/index.h index 16bacc2..3dac6fb 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/index.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/index.h @@ -303,7 +303,7 @@ extern LZMA_API(uint64_t) lzma_index_memused(const lzma_index *i) * \return On success, a pointer to an empty initialized lzma_index is * returned. If allocation fails, NULL is returned. */ -extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator) +extern LZMA_API(lzma_index *) lzma_index_init(const lzma_allocator *allocator) lzma_nothrow; @@ -312,8 +312,8 @@ extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator) * * If i is NULL, this does nothing. */ -extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator) - lzma_nothrow; +extern LZMA_API(void) lzma_index_end( + lzma_index *i, const lzma_allocator *allocator) lzma_nothrow; /** @@ -341,7 +341,7 @@ extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator) * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_index_append( - lzma_index *i, lzma_allocator *allocator, + lzma_index *i, const lzma_allocator *allocator, lzma_vli unpadded_size, lzma_vli uncompressed_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -564,8 +564,8 @@ extern LZMA_API(lzma_bool) lzma_index_iter_locate( * - LZMA_MEM_ERROR * - LZMA_PROG_ERROR */ -extern LZMA_API(lzma_ret) lzma_index_cat( - lzma_index *dest, lzma_index *src, lzma_allocator *allocator) +extern LZMA_API(lzma_ret) lzma_index_cat(lzma_index *dest, lzma_index *src, + const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -575,7 +575,7 @@ extern LZMA_API(lzma_ret) lzma_index_cat( * \return A copy of the lzma_index, or NULL if memory allocation failed. */ extern LZMA_API(lzma_index *) lzma_index_dup( - const lzma_index *i, lzma_allocator *allocator) + const lzma_index *i, const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -586,8 +586,7 @@ extern LZMA_API(lzma_index *) lzma_index_dup( * \param i Pointer to lzma_index which should be encoded. * * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. - * It is enough to use only one of them (you can choose freely; use LZMA_RUN - * to support liblzma versions older than 5.0.0). + * It is enough to use only one of them (you can choose freely). * * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR @@ -610,16 +609,21 @@ extern LZMA_API(lzma_ret) lzma_index_encoder( * to a new lzma_index, which the application * has to later free with lzma_index_end(). * \param memlimit How much memory the resulting lzma_index is - * allowed to require. + * allowed to require. liblzma 5.2.3 and earlier + * don't allow 0 here and return LZMA_PROG_ERROR; + * later versions treat 0 as if 1 had been specified. * - * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. - * It is enough to use only one of them (you can choose freely; use LZMA_RUN - * to support liblzma versions older than 5.0.0). + * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * There is no need to use LZMA_FINISH, but it's allowed because it may + * simplify certain types of applications. * * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR - * - LZMA_MEMLIMIT_ERROR * - LZMA_PROG_ERROR + * + * liblzma 5.2.3 and older list also LZMA_MEMLIMIT_ERROR here + * but that error code has never been possible from this + * initialization function. */ extern LZMA_API(lzma_ret) lzma_index_decoder( lzma_stream *strm, lzma_index **i, uint64_t memlimit) @@ -677,6 +681,6 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i, * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_index_buffer_decode(lzma_index **i, - uint64_t *memlimit, lzma_allocator *allocator, + uint64_t *memlimit, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h b/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h index fa2e048..9287f1d 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h @@ -37,7 +37,7 @@ typedef struct lzma_index_hash_s lzma_index_hash; * pointer than the index_hash that was given as an argument. */ extern LZMA_API(lzma_index_hash *) lzma_index_hash_init( - lzma_index_hash *index_hash, lzma_allocator *allocator) + lzma_index_hash *index_hash, const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -45,7 +45,7 @@ extern LZMA_API(lzma_index_hash *) lzma_index_hash_init( * \brief Deallocate lzma_index_hash structure */ extern LZMA_API(void) lzma_index_hash_end( - lzma_index_hash *index_hash, lzma_allocator *allocator) + lzma_index_hash *index_hash, const lzma_allocator *allocator) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/lzma.h b/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h index 3f8e095..4e32fa3 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/lzma.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h @@ -1,5 +1,5 @@ /** - * \file lzma/lzma.h + * \file lzma/lzma12.h * \brief LZMA1 and LZMA2 filters */ diff --git a/Utilities/cmliblzma/liblzma/api/lzma/version.h b/Utilities/cmliblzma/liblzma/api/lzma/version.h index 09866b9..143c7de 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/version.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/version.h @@ -21,8 +21,8 @@ * Version number split into components */ #define LZMA_VERSION_MAJOR 5 -#define LZMA_VERSION_MINOR 0 -#define LZMA_VERSION_PATCH 8 +#define LZMA_VERSION_MINOR 2 +#define LZMA_VERSION_PATCH 4 #define LZMA_VERSION_STABILITY LZMA_VERSION_STABILITY_STABLE #ifndef LZMA_VERSION_COMMIT diff --git a/Utilities/cmliblzma/liblzma/check/check.c b/Utilities/cmliblzma/liblzma/check/check.c index 979b0a8..428ddae 100644 --- a/Utilities/cmliblzma/liblzma/check/check.c +++ b/Utilities/cmliblzma/liblzma/check/check.c @@ -16,6 +16,9 @@ extern LZMA_API(lzma_bool) lzma_check_is_supported(lzma_check type) { + if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) + return false; + static const lzma_bool available_checks[LZMA_CHECK_ID_MAX + 1] = { true, // LZMA_CHECK_NONE @@ -53,9 +56,6 @@ lzma_check_is_supported(lzma_check type) false, // Reserved }; - if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) - return false; - return available_checks[(unsigned int)(type)]; } @@ -63,6 +63,9 @@ lzma_check_is_supported(lzma_check type) extern LZMA_API(uint32_t) lzma_check_size(lzma_check type) { + if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) + return UINT32_MAX; + // See file-format.txt section 2.1.1.2. static const uint8_t check_sizes[LZMA_CHECK_ID_MAX + 1] = { 0, @@ -73,9 +76,6 @@ lzma_check_size(lzma_check type) 64, 64, 64 }; - if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) - return UINT32_MAX; - return check_sizes[(unsigned int)(type)]; } diff --git a/Utilities/cmliblzma/liblzma/check/check.h b/Utilities/cmliblzma/liblzma/check/check.h index e100d2b..3007d88 100644 --- a/Utilities/cmliblzma/liblzma/check/check.h +++ b/Utilities/cmliblzma/liblzma/check/check.h @@ -15,6 +15,53 @@ #include "common.h" +// If the function for external SHA-256 is missing, use the internal SHA-256 +// code. Due to how configure works, these defines can only get defined when +// both a usable header and a type have already been found. +#if !(defined(HAVE_CC_SHA256_INIT) \ + || defined(HAVE_SHA256_INIT) \ + || defined(HAVE_SHA256INIT)) +# define HAVE_INTERNAL_SHA256 1 +#endif + +#if defined(HAVE_INTERNAL_SHA256) +// Nothing +#elif defined(HAVE_COMMONCRYPTO_COMMONDIGEST_H) +# include <CommonCrypto/CommonDigest.h> +#elif defined(HAVE_SHA256_H) +# include <sys/types.h> +# include <sha256.h> +#elif defined(HAVE_SHA2_H) +# include <sys/types.h> +# include <sha2.h> +#endif + +#if defined(HAVE_INTERNAL_SHA256) +/// State for the internal SHA-256 implementation +typedef struct { + /// Internal state + uint32_t state[8]; + + /// Size of the message excluding padding + uint64_t size; +} lzma_sha256_state; +#elif defined(HAVE_CC_SHA256_CTX) +typedef CC_SHA256_CTX lzma_sha256_state; +#elif defined(HAVE_SHA256_CTX) +typedef SHA256_CTX lzma_sha256_state; +#elif defined(HAVE_SHA2_CTX) +typedef SHA2_CTX lzma_sha256_state; +#endif + +#if defined(HAVE_INTERNAL_SHA256) +// Nothing +#elif defined(HAVE_CC_SHA256_INIT) +# define LZMA_SHA256FUNC(x) CC_SHA256_ ## x +#elif defined(HAVE_SHA256_INIT) +# define LZMA_SHA256FUNC(x) SHA256_ ## x +#elif defined(HAVE_SHA256INIT) +# define LZMA_SHA256FUNC(x) SHA256 ## x +#endif // Index hashing needs the best possible hash function (preferably // a cryptographic hash) for maximum reliability. @@ -43,14 +90,7 @@ typedef struct { union { uint32_t crc32; uint64_t crc64; - - struct { - /// Internal state - uint32_t state[8]; - - /// Size of the message excluding padding - uint64_t size; - } sha256; + lzma_sha256_state sha256; } state; } lzma_check_state; @@ -82,6 +122,8 @@ extern void lzma_check_update(lzma_check_state *check, lzma_check type, extern void lzma_check_finish(lzma_check_state *check, lzma_check type); +#ifndef LZMA_SHA256FUNC + /// Prepare SHA-256 state for new input. extern void lzma_sha256_init(lzma_check_state *check); @@ -92,4 +134,39 @@ extern void lzma_sha256_update( /// Finish the SHA-256 calculation and store the result to check->buffer.u8. extern void lzma_sha256_finish(lzma_check_state *check); + +#else + +static inline void +lzma_sha256_init(lzma_check_state *check) +{ + LZMA_SHA256FUNC(Init)(&check->state.sha256); +} + + +static inline void +lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) +{ +#if defined(HAVE_CC_SHA256_INIT) && SIZE_MAX > UINT32_MAX + // Darwin's CC_SHA256_Update takes uint32_t as the buffer size, + // so use a loop to support size_t. + while (size > UINT32_MAX) { + LZMA_SHA256FUNC(Update)(&check->state.sha256, buf, UINT32_MAX); + buf += UINT32_MAX; + size -= UINT32_MAX; + } +#endif + + LZMA_SHA256FUNC(Update)(&check->state.sha256, buf, size); +} + + +static inline void +lzma_sha256_finish(lzma_check_state *check) +{ + LZMA_SHA256FUNC(Final)(check->buffer.u8, &check->state.sha256); +} + +#endif + #endif diff --git a/Utilities/cmliblzma/liblzma/check/crc32_fast.c b/Utilities/cmliblzma/liblzma/check/crc32_fast.c index c2c3cb7..3de0263 100644 --- a/Utilities/cmliblzma/liblzma/check/crc32_fast.c +++ b/Utilities/cmliblzma/liblzma/check/crc32_fast.c @@ -33,8 +33,6 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) #endif if (size > 8) { - const uint8_t * limit; - // Fix the alignment, if needed. The if statement above // ensures that this won't read past the end of buf[]. while ((uintptr_t)(buf) & 7) { @@ -43,7 +41,7 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) } // Calculate the position where to stop. - limit = buf + (size & ~(size_t)(7)); + const uint8_t *const limit = buf + (size & ~(size_t)(7)); // Calculate how many bytes must be calculated separately // before returning the result. @@ -51,8 +49,6 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) // Calculate the CRC32 using the slice-by-eight algorithm. while (buf < limit) { - uint32_t tmp; - crc ^= *(const uint32_t *)(buf); buf += 4; @@ -61,7 +57,7 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) ^ lzma_crc32_table[5][C(crc)] ^ lzma_crc32_table[4][D(crc)]; - tmp = *(const uint32_t *)(buf); + const uint32_t tmp = *(const uint32_t *)(buf); buf += 4; // At least with some compilers, it is critical for diff --git a/Utilities/cmliblzma/liblzma/check/crc64_fast.c b/Utilities/cmliblzma/liblzma/check/crc64_fast.c index 1436557..52af29e 100644 --- a/Utilities/cmliblzma/liblzma/check/crc64_fast.c +++ b/Utilities/cmliblzma/liblzma/check/crc64_fast.c @@ -36,14 +36,12 @@ lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) #endif if (size > 4) { - const uint8_t *limit; - while ((uintptr_t)(buf) & 3) { crc = lzma_crc64_table[0][*buf++ ^ A1(crc)] ^ S8(crc); --size; } - limit = buf + (size & ~(size_t)(3)); + const uint8_t *const limit = buf + (size & ~(size_t)(3)); size &= (size_t)(3); while (buf < limit) { diff --git a/Utilities/cmliblzma/liblzma/check/sha256.c b/Utilities/cmliblzma/liblzma/check/sha256.c index 3af6aa6..5eede5c 100644 --- a/Utilities/cmliblzma/liblzma/check/sha256.c +++ b/Utilities/cmliblzma/liblzma/check/sha256.c @@ -21,22 +21,22 @@ // /////////////////////////////////////////////////////////////////////////////// -// Avoid bogus warnings in transform(). -#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 2) || __GNUC__ > 4 -# pragma GCC diagnostic ignored "-Wuninitialized" -#endif - #include "check.h" -// At least on x86, GCC is able to optimize this to a rotate instruction. -#define rotr_32(num, amount) ((num) >> (amount) | (num) << (32 - (amount))) +// Rotate a uint32_t. GCC can optimize this to a rotate instruction +// at least on x86. +static inline uint32_t +rotr_32(uint32_t num, unsigned amount) +{ + return (num >> amount) | (num << (32 - amount)); +} -#define blk0(i) (W[i] = data[i]) +#define blk0(i) (W[i] = conv32be(data[i])) #define blk2(i) (W[i & 15] += s1(W[(i - 2) & 15]) + W[(i - 7) & 15] \ + s0(W[(i - 15) & 15])) #define Ch(x, y, z) (z ^ (x & (y ^ z))) -#define Maj(x, y, z) ((x & y) | (z & (x | y))) +#define Maj(x, y, z) ((x & (y ^ z)) + (y & z)) #define a(i) T[(0 - i) & 7] #define b(i) T[(1 - i) & 7] @@ -47,16 +47,17 @@ #define g(i) T[(6 - i) & 7] #define h(i) T[(7 - i) & 7] -#define R(i) \ - h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] \ - + (j ? blk2(i) : blk0(i)); \ +#define R(i, j, blk) \ + h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] + blk; \ d(i) += h(i); \ h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) +#define R0(i) R(i, 0, blk0(i)) +#define R2(i) R(i, j, blk2(i)) -#define S0(x) (rotr_32(x, 2) ^ rotr_32(x, 13) ^ rotr_32(x, 22)) -#define S1(x) (rotr_32(x, 6) ^ rotr_32(x, 11) ^ rotr_32(x, 25)) -#define s0(x) (rotr_32(x, 7) ^ rotr_32(x, 18) ^ (x >> 3)) -#define s1(x) (rotr_32(x, 17) ^ rotr_32(x, 19) ^ (x >> 10)) +#define S0(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 9), 11), 2) +#define S1(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 14), 5), 6) +#define s0(x) (rotr_32(x ^ rotr_32(x, 11), 7) ^ (x >> 3)) +#define s1(x) (rotr_32(x ^ rotr_32(x, 2), 17) ^ (x >> 10)) static const uint32_t SHA256_K[64] = { @@ -84,17 +85,22 @@ transform(uint32_t state[8], const uint32_t data[16]) { uint32_t W[16]; uint32_t T[8]; - unsigned int j; // Copy state[] to working vars. memcpy(T, state, sizeof(T)); - // 64 operations, partially loop unrolled - for (j = 0; j < 64; j += 16) { - R( 0); R( 1); R( 2); R( 3); - R( 4); R( 5); R( 6); R( 7); - R( 8); R( 9); R(10); R(11); - R(12); R(13); R(14); R(15); + // The first 16 operations unrolled + R0( 0); R0( 1); R0( 2); R0( 3); + R0( 4); R0( 5); R0( 6); R0( 7); + R0( 8); R0( 9); R0(10); R0(11); + R0(12); R0(13); R0(14); R0(15); + + // The remaining 48 operations partially unrolled + for (unsigned int j = 16; j < 64; j += 16) { + R2( 0); R2( 1); R2( 2); R2( 3); + R2( 4); R2( 5); R2( 6); R2( 7); + R2( 8); R2( 9); R2(10); R2(11); + R2(12); R2(13); R2(14); R2(15); } // Add the working vars back into state[]. @@ -112,19 +118,7 @@ transform(uint32_t state[8], const uint32_t data[16]) static void process(lzma_check_state *check) { -#ifdef WORDS_BIGENDIAN transform(check->state.sha256.state, check->buffer.u32); - -#else - uint32_t data[16]; - size_t i; - - for (i = 0; i < 16; ++i) - data[i] = bswap32(check->buffer.u32[i]); - - transform(check->state.sha256.state, data); -#endif - return; } @@ -174,8 +168,6 @@ lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) extern void lzma_sha256_finish(lzma_check_state *check) { - size_t i; - // Add padding as described in RFC 3174 (it describes SHA-1 but // the same padding style is used for SHA-256 too). size_t pos = check->state.sha256.size & 0x3F; @@ -197,7 +189,7 @@ lzma_sha256_finish(lzma_check_state *check) process(check); - for (i = 0; i < 8; ++i) + for (size_t i = 0; i < 8; ++i) check->buffer.u32[i] = conv32be(check->state.sha256.state[i]); return; diff --git a/Utilities/cmliblzma/liblzma/common/alone_decoder.c b/Utilities/cmliblzma/liblzma/common/alone_decoder.c index 5f5e564..77d0a9b 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/alone_decoder.c @@ -15,7 +15,7 @@ #include "lz_decoder.h" -struct lzma_coder_s { +typedef struct { lzma_next_coder next; enum { @@ -46,17 +46,19 @@ struct lzma_coder_s { /// Options decoded from the header needed to initialize /// the LZMA decoder lzma_options_lzma options; -}; +} lzma_alone_coder; static lzma_ret -alone_decode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +alone_decode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_alone_coder *coder = coder_ptr; + while (*out_pos < out_size && (coder->sequence == SEQ_CODE || *in_pos < in_size)) switch (coder->sequence) { @@ -126,17 +128,19 @@ alone_decode(lzma_coder *coder, // Fall through case SEQ_CODER_INIT: { - lzma_ret ret; + if (coder->memusage > coder->memlimit) + return LZMA_MEMLIMIT_ERROR; lzma_filter_info filters[2] = { - { 0, &lzma_lzma_decoder_init, &coder->options }, - { 0, NULL, NULL } + { + .init = &lzma_lzma_decoder_init, + .options = &coder->options, + }, { + .init = NULL, + } }; - if (coder->memusage > coder->memlimit) - return LZMA_MEMLIMIT_ERROR; - - ret = lzma_next_filter_init(&coder->next, + const lzma_ret ret = lzma_next_filter_init(&coder->next, allocator, filters); if (ret != LZMA_OK) return ret; @@ -164,8 +168,9 @@ alone_decode(lzma_coder *coder, static void -alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +alone_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_alone_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -173,9 +178,11 @@ alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -alone_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +alone_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_alone_coder *coder = coder_ptr; + *memusage = coder->memusage; *old_memlimit = coder->memlimit; @@ -191,34 +198,34 @@ alone_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, extern lzma_ret -lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_alone_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, bool picky) { lzma_next_coder_init(&lzma_alone_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; + lzma_alone_coder *coder = next->coder; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_alone_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &alone_decode; next->end = &alone_decoder_end; next->memconfig = &alone_decoder_memconfig; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } - next->coder->sequence = SEQ_PROPERTIES; - next->coder->picky = picky; - next->coder->pos = 0; - next->coder->options.dict_size = 0; - next->coder->options.preset_dict = NULL; - next->coder->options.preset_dict_size = 0; - next->coder->uncompressed_size = 0; - next->coder->memlimit = memlimit; - next->coder->memusage = LZMA_MEMUSAGE_BASE; + coder->sequence = SEQ_PROPERTIES; + coder->picky = picky; + coder->pos = 0; + coder->options.dict_size = 0; + coder->options.preset_dict = NULL; + coder->options.preset_dict_size = 0; + coder->uncompressed_size = 0; + coder->memlimit = my_max(1, memlimit); + coder->memusage = LZMA_MEMUSAGE_BASE; return LZMA_OK; } @@ -227,7 +234,7 @@ lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit) { - lzma_next_strm_init2(lzma_alone_decoder_init, strm, memlimit, false); + lzma_next_strm_init(lzma_alone_decoder_init, strm, memlimit, false); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/alone_decoder.h b/Utilities/cmliblzma/liblzma/common/alone_decoder.h index f666fc3..dfa031a 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/alone_decoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_alone_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, bool picky); #endif diff --git a/Utilities/cmliblzma/liblzma/common/alone_encoder.c b/Utilities/cmliblzma/liblzma/common/alone_encoder.c index 4207b4a..4853cfd 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/alone_encoder.c @@ -17,7 +17,7 @@ #define ALONE_HEADER_SIZE (1 + 4 + 8) -struct lzma_coder_s { +typedef struct { lzma_next_coder next; enum { @@ -27,17 +27,19 @@ struct lzma_coder_s { size_t header_pos; uint8_t header[ALONE_HEADER_SIZE]; -}; +} lzma_alone_coder; static lzma_ret -alone_encode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +alone_encode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_alone_coder *coder = coder_ptr; + while (*out_pos < out_size) switch (coder->sequence) { case SEQ_HEADER: @@ -65,8 +67,9 @@ alone_encode(lzma_coder *coder, static void -alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +alone_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_alone_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -75,36 +78,31 @@ alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) // At least for now, this is not used by any internal function. static lzma_ret -alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_options_lzma *options) { - uint32_t d; - - // Initialize the LZMA encoder. - const lzma_filter_info filters[2] = { - { 0, &lzma_lzma_encoder_init, (void *)(options) }, - { 0, NULL, NULL } - }; - lzma_next_coder_init(&alone_encoder_init, next, allocator); - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_alone_coder *coder = next->coder; + + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_alone_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &alone_encode; next->end = &alone_encoder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_HEADER; - next->coder->header_pos = 0; + coder->sequence = SEQ_HEADER; + coder->header_pos = 0; // Encode the header: // - Properties (1 byte) - if (lzma_lzma_lclppb_encode(options, next->coder->header)) + if (lzma_lzma_lclppb_encode(options, coder->header)) return LZMA_OPTIONS_ERROR; // - Dictionary size (4 bytes) @@ -115,7 +113,7 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // one is the next unless it is UINT32_MAX. While the header would // allow any 32-bit integer, we do this to keep the decoder of liblzma // accepting the resulting files. - d = options->dict_size - 1; + uint32_t d = options->dict_size - 1; d |= d >> 2; d |= d >> 3; d |= d >> 4; @@ -124,18 +122,28 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, if (d != UINT32_MAX) ++d; - unaligned_write32le(next->coder->header + 1, d); + unaligned_write32le(coder->header + 1, d); // - Uncompressed size (always unknown and using EOPM) - memset(next->coder->header + 1 + 4, 0xFF, 8); + memset(coder->header + 1 + 4, 0xFF, 8); + + // Initialize the LZMA encoder. + const lzma_filter_info filters[2] = { + { + .init = &lzma_lzma_encoder_init, + .options = (void *)(options), + }, { + .init = NULL, + } + }; - return lzma_next_filter_init(&next->coder->next, allocator, filters); + return lzma_next_filter_init(&coder->next, allocator, filters); } /* extern lzma_ret -lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_options_alone *options) { lzma_next_coder_init(&alone_encoder_init, next, allocator, options); @@ -146,7 +154,7 @@ lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_alone_encoder(lzma_stream *strm, const lzma_options_lzma *options) { - lzma_next_strm_init1(alone_encoder_init, strm, options); + lzma_next_strm_init(alone_encoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/auto_decoder.c b/Utilities/cmliblzma/liblzma/common/auto_decoder.c index 24cf489..6895c7c 100644 --- a/Utilities/cmliblzma/liblzma/common/auto_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/auto_decoder.c @@ -14,7 +14,7 @@ #include "alone_decoder.h" -struct lzma_coder_s { +typedef struct { /// Stream decoder or LZMA_Alone decoder lzma_next_coder next; @@ -26,15 +26,17 @@ struct lzma_coder_s { SEQ_CODE, SEQ_FINISH, } sequence; -}; +} lzma_auto_coder; static lzma_ret -auto_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +auto_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_auto_coder *coder = coder_ptr; + switch (coder->sequence) { case SEQ_INIT: if (*in_pos >= in_size) @@ -100,8 +102,9 @@ auto_decode(lzma_coder *coder, lzma_allocator *allocator, static void -auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +auto_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_auto_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -109,8 +112,10 @@ auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_check -auto_decoder_get_check(const lzma_coder *coder) +auto_decoder_get_check(const void *coder_ptr) { + const lzma_auto_coder *coder = coder_ptr; + // It is LZMA_Alone if get_check is NULL. return coder->next.get_check == NULL ? LZMA_CHECK_NONE : coder->next.get_check(coder->next.coder); @@ -118,9 +123,11 @@ auto_decoder_get_check(const lzma_coder *coder) static lzma_ret -auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +auto_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_auto_coder *coder = coder_ptr; + lzma_ret ret; if (coder->next.memconfig != NULL) { @@ -132,7 +139,10 @@ auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, // the current memory usage. *memusage = LZMA_MEMUSAGE_BASE; *old_memlimit = coder->memlimit; + ret = LZMA_OK; + if (new_memlimit != 0 && new_memlimit < *memusage) + ret = LZMA_MEMLIMIT_ERROR; } if (ret == LZMA_OK && new_memlimit != 0) @@ -143,32 +153,31 @@ auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, static lzma_ret -auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +auto_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, uint32_t flags) { lzma_next_coder_init(&auto_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; - if (flags & ~LZMA_SUPPORTED_FLAGS) return LZMA_OPTIONS_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_auto_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_auto_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &auto_decode; next->end = &auto_decoder_end; next->get_check = &auto_decoder_get_check; next->memconfig = &auto_decoder_memconfig; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } - next->coder->memlimit = memlimit; - next->coder->flags = flags; - next->coder->sequence = SEQ_INIT; + coder->memlimit = my_max(1, memlimit); + coder->flags = flags; + coder->sequence = SEQ_INIT; return LZMA_OK; } @@ -177,7 +186,7 @@ auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_auto_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init2(auto_decoder_init, strm, memlimit, flags); + lzma_next_strm_init(auto_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c index b4bd388..b0ded90 100644 --- a/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c @@ -14,13 +14,10 @@ extern LZMA_API(lzma_ret) -lzma_block_buffer_decode(lzma_block *block, lzma_allocator *allocator, +lzma_block_buffer_decode(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder block_decoder; - lzma_ret ret; - if (in_pos == NULL || (in == NULL && *in_pos != in_size) || *in_pos > in_size || out_pos == NULL || (out == NULL && *out_pos != out_size) @@ -28,8 +25,9 @@ lzma_block_buffer_decode(lzma_block *block, lzma_allocator *allocator, return LZMA_PROG_ERROR; // Initialize the Block decoder. - block_decoder = LZMA_NEXT_CODER_INIT; - ret = lzma_block_decoder_init(&block_decoder, allocator, block); + lzma_next_coder block_decoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_block_decoder_init( + &block_decoder, allocator, block); if (ret == LZMA_OK) { // Save the positions so that we can restore them in case diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c index 136f7f5..39e263a 100644 --- a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c @@ -10,6 +10,7 @@ // /////////////////////////////////////////////////////////////////////////////// +#include "block_buffer_encoder.h" #include "block_encoder.h" #include "filter_encoder.h" #include "lzma2_encoder.h" @@ -28,11 +29,9 @@ + LZMA_CHECK_SIZE_MAX + 3) & ~3) -static lzma_vli -lzma2_bound(lzma_vli uncompressed_size) +static uint64_t +lzma2_bound(uint64_t uncompressed_size) { - lzma_vli overhead; - // Prevent integer overflow in overhead calculation. if (uncompressed_size > COMPRESSED_SIZE_MAX) return 0; @@ -41,7 +40,7 @@ lzma2_bound(lzma_vli uncompressed_size) // uncompressed_size up to the next multiple of LZMA2_CHUNK_MAX, // multiply by the size of per-chunk header, and add one byte for // the end marker. - overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1) + const uint64_t overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1) / LZMA2_CHUNK_MAX) * LZMA2_HEADER_UNCOMPRESSED + 1; @@ -53,30 +52,36 @@ lzma2_bound(lzma_vli uncompressed_size) } -extern LZMA_API(size_t) -lzma_block_buffer_bound(size_t uncompressed_size) +extern uint64_t +lzma_block_buffer_bound64(uint64_t uncompressed_size) { - // For now, if the data doesn't compress, we always use uncompressed - // chunks of LZMA2. In future we may use Subblock filter too, but - // but for simplicity we probably will still use the same bound - // calculation even though Subblock filter would have slightly less - // overhead. - lzma_vli lzma2_size = lzma2_bound(uncompressed_size); + // If the data doesn't compress, we always use uncompressed + // LZMA2 chunks. + uint64_t lzma2_size = lzma2_bound(uncompressed_size); if (lzma2_size == 0) return 0; // Take Block Padding into account. - lzma2_size = (lzma2_size + 3) & ~LZMA_VLI_C(3); + lzma2_size = (lzma2_size + 3) & ~UINT64_C(3); -#if SIZE_MAX < LZMA_VLI_MAX - // Catch the possible integer overflow on 32-bit systems. There's no - // overflow on 64-bit systems, because lzma2_bound() already takes + // No risk of integer overflow because lzma2_bound() already takes // into account the size of the headers in the Block. - if (SIZE_MAX - HEADERS_BOUND < lzma2_size) + return HEADERS_BOUND + lzma2_size; +} + + +extern LZMA_API(size_t) +lzma_block_buffer_bound(size_t uncompressed_size) +{ + uint64_t ret = lzma_block_buffer_bound64(uncompressed_size); + +#if SIZE_MAX < UINT64_MAX + // Catch the possible integer overflow on 32-bit systems. + if (ret > SIZE_MAX) return 0; #endif - return HEADERS_BOUND + lzma2_size; + return ret; } @@ -84,17 +89,12 @@ static lzma_ret block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - size_t in_pos = 0; - uint8_t control = 0x01; // Dictionary reset - lzma_filter *filters_orig; - - // TODO: Figure out if the last filter is LZMA2 or Subblock and use - // that filter to encode the uncompressed chunks. - // Use LZMA2 uncompressed chunks. We wouldn't need a dictionary at // all, but LZMA2 always requires a dictionary, so use the minimum // value to minimize memory usage of the decoder. - lzma_options_lzma lzma2 = { LZMA_DICT_SIZE_MIN }; + lzma_options_lzma lzma2 = { + .dict_size = LZMA_DICT_SIZE_MIN, + }; lzma_filter filters[2]; filters[0].id = LZMA_FILTER_LZMA2; @@ -103,7 +103,7 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, // Set the above filter options to *block temporarily so that we can // encode the Block Header. - filters_orig = block->filters; + lzma_filter *filters_orig = block->filters; block->filters = filters; if (lzma_block_header_size(block) != LZMA_OK) { @@ -132,17 +132,18 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, *out_pos += block->header_size; // Encode the data using LZMA2 uncompressed chunks. + size_t in_pos = 0; + uint8_t control = 0x01; // Dictionary reset while (in_pos < in_size) { - size_t copy_size; - // Control byte: Indicate uncompressed chunk, of which // the first resets the dictionary. out[(*out_pos)++] = control; control = 0x02; // No dictionary reset // Size of the uncompressed chunk - copy_size = my_min(in_size - in_pos, LZMA2_CHUNK_MAX); + const size_t copy_size + = my_min(in_size - in_pos, LZMA2_CHUNK_MAX); out[(*out_pos)++] = (copy_size - 1) >> 8; out[(*out_pos)++] = (copy_size - 1) & 0xFF; @@ -163,27 +164,18 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, static lzma_ret -block_encode_normal(lzma_block *block, lzma_allocator *allocator, +block_encode_normal(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - size_t out_start; - lzma_next_coder raw_encoder = LZMA_NEXT_CODER_INIT; - lzma_ret ret; - // Find out the size of the Block Header. - block->compressed_size = lzma2_bound(in_size); - if (block->compressed_size == 0) - return LZMA_DATA_ERROR; - - block->uncompressed_size = in_size; return_if_error(lzma_block_header_size(block)); // Reserve space for the Block Header and skip it for now. if (out_size - *out_pos <= block->header_size) return LZMA_BUF_ERROR; - out_start = *out_pos; + const size_t out_start = *out_pos; *out_pos += block->header_size; // Limit out_size so that we stop encoding if the output would grow @@ -193,7 +185,8 @@ block_encode_normal(lzma_block *block, lzma_allocator *allocator, // TODO: In many common cases this could be optimized to use // significantly less memory. - ret = lzma_raw_encoder_init( + lzma_next_coder raw_encoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_raw_encoder_init( &raw_encoder, allocator, block->filters); if (ret == LZMA_OK) { @@ -227,15 +220,12 @@ block_encode_normal(lzma_block *block, lzma_allocator *allocator, } -extern LZMA_API(lzma_ret) -lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, +static lzma_ret +block_buffer_encode(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, - uint8_t *out, size_t *out_pos, size_t out_size) + uint8_t *out, size_t *out_pos, size_t out_size, + bool try_to_compress) { - size_t check_size; - lzma_ret ret; - size_t i; - // Validate the arguments. if (block == NULL || (in == NULL && in_size != 0) || out == NULL || out_pos == NULL || *out_pos > out_size) @@ -243,11 +233,11 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, // The contents of the structure may depend on the version so // check the version before validating the contents of *block. - if (block->version != 0) + if (block->version > 1) return LZMA_OPTIONS_ERROR; if ((unsigned int)(block->check) > LZMA_CHECK_ID_MAX - || block->filters == NULL) + || (try_to_compress && block->filters == NULL)) return LZMA_PROG_ERROR; if (!lzma_check_is_supported(block->check)) @@ -259,7 +249,7 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, out_size -= (out_size - *out_pos) & 3; // Get the size of the Check field. - check_size = lzma_check_size(block->check); + const size_t check_size = lzma_check_size(block->check); assert(check_size != UINT32_MAX); // Reserve space for the Check field. @@ -268,9 +258,19 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, out_size -= check_size; + // Initialize block->uncompressed_size and calculate the worst-case + // value for block->compressed_size. + block->uncompressed_size = in_size; + block->compressed_size = lzma2_bound(in_size); + if (block->compressed_size == 0) + return LZMA_DATA_ERROR; + // Do the actual compression. - ret = block_encode_normal(block, allocator, - in, in_size, out, out_pos, out_size); + lzma_ret ret = LZMA_BUF_ERROR; + if (try_to_compress) + ret = block_encode_normal(block, allocator, + in, in_size, out, out_pos, out_size); + if (ret != LZMA_OK) { // If the error was something else than output buffer // becoming full, return the error now. @@ -291,7 +291,7 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, // Block Padding. No buffer overflow here, because we already adjusted // out_size so that (out_size - out_start) is a multiple of four. // Thus, if the buffer is full, the loop body can never run. - for (i = (size_t)(block->compressed_size); i & 3; ++i) { + for (size_t i = (size_t)(block->compressed_size); i & 3; ++i) { assert(*out_pos < out_size); out[(*out_pos)++] = 0x00; } @@ -313,3 +313,25 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, return LZMA_OK; } + + +extern LZMA_API(lzma_ret) +lzma_block_buffer_encode(lzma_block *block, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + return block_buffer_encode(block, allocator, + in, in_size, out, out_pos, out_size, true); +} + + +extern LZMA_API(lzma_ret) +lzma_block_uncomp_encode(lzma_block *block, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + // It won't allocate any memory from heap so no need + // for lzma_allocator. + return block_buffer_encode(block, NULL, + in, in_size, out, out_pos, out_size, false); +} diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h new file mode 100644 index 0000000..653207f --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h @@ -0,0 +1,24 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_buffer_encoder.h +/// \brief Single-call .xz Block encoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BLOCK_BUFFER_ENCODER_H +#define LZMA_BLOCK_BUFFER_ENCODER_H + +#include "common.h" + + +/// uint64_t version of lzma_block_buffer_bound(). It is used by +/// stream_encoder_mt.c. Probably the original lzma_block_buffer_bound() +/// should have been 64-bit, but fixing it would break the ABI. +extern uint64_t lzma_block_buffer_bound64(uint64_t uncompressed_size); + +#endif diff --git a/Utilities/cmliblzma/liblzma/common/block_decoder.c b/Utilities/cmliblzma/liblzma/common/block_decoder.c index 35996e7..075bd27 100644 --- a/Utilities/cmliblzma/liblzma/common/block_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_decoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_CODE, SEQ_PADDING, @@ -45,7 +45,10 @@ struct lzma_coder_s { /// Check of the uncompressed data lzma_check_state check; -}; + + /// True if the integrity check won't be calculated and verified. + bool ignore_check; +} lzma_block_coder; static inline bool @@ -71,11 +74,13 @@ is_size_valid(lzma_vli size, lzma_vli reference) static lzma_ret -block_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +block_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_block_coder *coder = coder_ptr; + switch (coder->sequence) { case SEQ_CODE: { const size_t in_start = *in_pos; @@ -97,8 +102,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, coder->block->uncompressed_size)) return LZMA_DATA_ERROR; - lzma_check_update(&coder->check, coder->block->check, - out + out_start, out_used); + if (!coder->ignore_check) + lzma_check_update(&coder->check, coder->block->check, + out + out_start, out_used); if (ret != LZMA_STREAM_END) return ret; @@ -140,7 +146,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, if (coder->block->check == LZMA_CHECK_NONE) return LZMA_STREAM_END; - lzma_check_finish(&coder->check, coder->block->check); + if (!coder->ignore_check) + lzma_check_finish(&coder->check, coder->block->check); + coder->sequence = SEQ_CHECK; // Fall through @@ -155,7 +163,8 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, // Validate the Check only if we support it. // coder->check.buffer may be uninitialized // when the Check ID is not supported. - if (lzma_check_is_supported(coder->block->check) + if (!coder->ignore_check + && lzma_check_is_supported(coder->block->check) && memcmp(coder->block->raw_check, coder->check.buffer.u8, check_size) != 0) @@ -170,8 +179,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, static void -block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +block_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_block_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -179,7 +189,7 @@ block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_block_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_block *block) { lzma_next_coder_init(&lzma_block_decoder_init, next, allocator); @@ -191,27 +201,29 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, || !lzma_vli_is_valid(block->uncompressed_size)) return LZMA_PROG_ERROR; - // Allocate and initialize *next->coder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + // Allocate *next->coder if needed. + lzma_block_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_block_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &block_decode; next->end = &block_decoder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_CODE; - next->coder->block = block; - next->coder->compressed_size = 0; - next->coder->uncompressed_size = 0; + coder->sequence = SEQ_CODE; + coder->block = block; + coder->compressed_size = 0; + coder->uncompressed_size = 0; // If Compressed Size is not known, we calculate the maximum allowed // value so that encoded size of the Block (including Block Padding) // is still a valid VLI and a multiple of four. - next->coder->compressed_limit + coder->compressed_limit = block->compressed_size == LZMA_VLI_UNKNOWN ? (LZMA_VLI_MAX & ~LZMA_VLI_C(3)) - block->header_size @@ -221,11 +233,14 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, // Initialize the check. It's caller's problem if the Check ID is not // supported, and the Block decoder cannot verify the Check field. // Caller can test lzma_check_is_supported(block->check). - next->coder->check_pos = 0; - lzma_check_init(&next->coder->check, block->check); + coder->check_pos = 0; + lzma_check_init(&coder->check, block->check); + + coder->ignore_check = block->version >= 1 + ? block->ignore_check : false; // Initialize the filter chain. - return lzma_raw_decoder_init(&next->coder->next, allocator, + return lzma_raw_decoder_init(&coder->next, allocator, block->filters); } @@ -233,7 +248,7 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_block_decoder(lzma_stream *strm, lzma_block *block) { - lzma_next_strm_init1(lzma_block_decoder_init, strm, block); + lzma_next_strm_init(lzma_block_decoder_init, strm, block); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_decoder.h b/Utilities/cmliblzma/liblzma/common/block_decoder.h index 7da9df6..718c5ce 100644 --- a/Utilities/cmliblzma/liblzma/common/block_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/block_decoder.h @@ -17,6 +17,6 @@ extern lzma_ret lzma_block_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_block *block); + const lzma_allocator *allocator, lzma_block *block); #endif diff --git a/Utilities/cmliblzma/liblzma/common/block_encoder.c b/Utilities/cmliblzma/liblzma/common/block_encoder.c index ed74827..168846a 100644 --- a/Utilities/cmliblzma/liblzma/common/block_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_encoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { /// The filters in the chain; initialized with lzma_raw_decoder_init(). lzma_next_coder next; @@ -41,15 +41,17 @@ struct lzma_coder_s { /// Check of the uncompressed data lzma_check_state check; -}; +} lzma_block_coder; static lzma_ret -block_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +block_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_block_coder *coder = coder_ptr; + // Check that our amount of input stays in proper limits. if (LZMA_VLI_MAX - coder->uncompressed_size < in_size - *in_pos) return LZMA_DATA_ERROR; @@ -134,8 +136,9 @@ block_encode(lzma_coder *coder, lzma_allocator *allocator, static void -block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +block_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_block_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -143,10 +146,12 @@ block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -block_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +block_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_block_coder *coder = coder_ptr; + if (coder->sequence != SEQ_CODE) return LZMA_PROG_ERROR; @@ -156,7 +161,7 @@ block_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_block_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_block *block) { lzma_next_coder_init(&lzma_block_encoder_init, next, allocator); @@ -166,7 +171,7 @@ lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // The contents of the structure may depend on the version so // check the version first. - if (block->version != 0) + if (block->version > 1) return LZMA_OPTIONS_ERROR; // If the Check ID is not supported, we cannot calculate the check and @@ -178,37 +183,38 @@ lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_UNSUPPORTED_CHECK; // Allocate and initialize *next->coder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_block_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_block_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &block_encode; next->end = &block_encoder_end; next->update = &block_encoder_update; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_CODE; - next->coder->block = block; - next->coder->compressed_size = 0; - next->coder->uncompressed_size = 0; - next->coder->pos = 0; + coder->sequence = SEQ_CODE; + coder->block = block; + coder->compressed_size = 0; + coder->uncompressed_size = 0; + coder->pos = 0; // Initialize the check - lzma_check_init(&next->coder->check, block->check); + lzma_check_init(&coder->check, block->check); // Initialize the requested filters. - return lzma_raw_encoder_init(&next->coder->next, allocator, - block->filters); + return lzma_raw_encoder_init(&coder->next, allocator, block->filters); } extern LZMA_API(lzma_ret) lzma_block_encoder(lzma_stream *strm, lzma_block *block) { - lzma_next_strm_init1(lzma_block_encoder_init, strm, block); + lzma_next_strm_init(lzma_block_encoder_init, strm, block); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_encoder.h b/Utilities/cmliblzma/liblzma/common/block_encoder.h index b9eff0b..bd97c18 100644 --- a/Utilities/cmliblzma/liblzma/common/block_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/block_encoder.h @@ -42,6 +42,6 @@ extern lzma_ret lzma_block_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_block *block); + const lzma_allocator *allocator, lzma_block *block); #endif diff --git a/Utilities/cmliblzma/liblzma/common/block_header_decoder.c b/Utilities/cmliblzma/liblzma/common/block_header_decoder.c index f6e470e..1dd982f 100644 --- a/Utilities/cmliblzma/liblzma/common/block_header_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_header_decoder.c @@ -15,14 +15,12 @@ static void -free_properties(lzma_block *block, lzma_allocator *allocator) +free_properties(lzma_block *block, const lzma_allocator *allocator) { - size_t i; - // Free allocated filter options. The last array member is not // touched after the initialization in the beginning of // lzma_block_header_decode(), so we don't need to touch that here. - for (i = 0; i < LZMA_FILTERS_MAX; ++i) { + for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) { lzma_free(block->filters[i].options, allocator); block->filters[i].id = LZMA_VLI_UNKNOWN; block->filters[i].options = NULL; @@ -34,15 +32,8 @@ free_properties(lzma_block *block, lzma_allocator *allocator) extern LZMA_API(lzma_ret) lzma_block_header_decode(lzma_block *block, - lzma_allocator *allocator, const uint8_t *in) + const lzma_allocator *allocator, const uint8_t *in) { - const size_t filter_count = (in[1] & 3) + 1; - size_t in_size; - size_t i; - - // Start after the Block Header Size and Block Flags fields. - size_t in_pos = 2; - // NOTE: We consider the header to be corrupt not only when the // CRC32 doesn't match, but also when variable-length integers // are invalid or over 63 bits, or if the header is too small @@ -50,13 +41,21 @@ lzma_block_header_decode(lzma_block *block, // Initialize the filter options array. This way the caller can // safely free() the options even if an error occurs in this function. - for (i = 0; i <= LZMA_FILTERS_MAX; ++i) { + for (size_t i = 0; i <= LZMA_FILTERS_MAX; ++i) { block->filters[i].id = LZMA_VLI_UNKNOWN; block->filters[i].options = NULL; } - // Always zero for now. - block->version = 0; + // Versions 0 and 1 are supported. If a newer version was specified, + // we need to downgrade it. + if (block->version > 1) + block->version = 1; + + // This isn't a Block Header option, but since the decompressor will + // read it if version >= 1, it's better to initialize it here than + // to expect the caller to do it since in almost all cases this + // should be false. + block->ignore_check = false; // Validate Block Header Size and Check type. The caller must have // already set these, so it is a programming error if this test fails. @@ -65,7 +64,7 @@ lzma_block_header_decode(lzma_block *block, return LZMA_PROG_ERROR; // Exclude the CRC32 field. - in_size = block->header_size - 4; + const size_t in_size = block->header_size - 4; // Verify CRC32 if (lzma_crc32(in, in_size, 0) != unaligned_read32le(in + in_size)) @@ -75,6 +74,9 @@ lzma_block_header_decode(lzma_block *block, if (in[1] & 0x3C) return LZMA_OPTIONS_ERROR; + // Start after the Block Header Size and Block Flags fields. + size_t in_pos = 2; + // Compressed Size if (in[1] & 0x40) { return_if_error(lzma_vli_decode(&block->compressed_size, @@ -96,7 +98,8 @@ lzma_block_header_decode(lzma_block *block, block->uncompressed_size = LZMA_VLI_UNKNOWN; // Filter Flags - for (i = 0; i < filter_count; ++i) { + const size_t filter_count = (in[1] & 3) + 1; + for (size_t i = 0; i < filter_count; ++i) { const lzma_ret ret = lzma_filter_flags_decode( &block->filters[i], allocator, in, &in_pos, in_size); diff --git a/Utilities/cmliblzma/liblzma/common/block_header_encoder.c b/Utilities/cmliblzma/liblzma/common/block_header_encoder.c index 650295c..5c5f542 100644 --- a/Utilities/cmliblzma/liblzma/common/block_header_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_header_encoder.c @@ -17,14 +17,12 @@ extern LZMA_API(lzma_ret) lzma_block_header_size(lzma_block *block) { - size_t i; + if (block->version > 1) + return LZMA_OPTIONS_ERROR; // Block Header Size + Block Flags + CRC32. uint32_t size = 1 + 1 + 4; - if (block->version != 0) - return LZMA_OPTIONS_ERROR; - // Compressed Size if (block->compressed_size != LZMA_VLI_UNKNOWN) { const uint32_t add = lzma_vli_size(block->compressed_size); @@ -47,13 +45,12 @@ lzma_block_header_size(lzma_block *block) if (block->filters == NULL || block->filters[0].id == LZMA_VLI_UNKNOWN) return LZMA_PROG_ERROR; - for (i = 0; block->filters[i].id != LZMA_VLI_UNKNOWN; ++i) { - uint32_t add; - + for (size_t i = 0; block->filters[i].id != LZMA_VLI_UNKNOWN; ++i) { // Don't allow too many filters. if (i == LZMA_FILTERS_MAX) return LZMA_PROG_ERROR; + uint32_t add; return_if_error(lzma_filter_flags_size(&add, block->filters + i)); @@ -76,23 +73,20 @@ lzma_block_header_size(lzma_block *block) extern LZMA_API(lzma_ret) lzma_block_header_encode(const lzma_block *block, uint8_t *out) { - size_t out_size; - size_t out_pos = 2; - size_t filter_count = 0; - // Validate everything but filters. if (lzma_block_unpadded_size(block) == 0 || !lzma_vli_is_valid(block->uncompressed_size)) return LZMA_PROG_ERROR; // Indicate the size of the buffer _excluding_ the CRC32 field. - out_size = block->header_size - 4; + const size_t out_size = block->header_size - 4; // Store the Block Header Size. out[0] = out_size / 4; // We write Block Flags in pieces. out[1] = 0x00; + size_t out_pos = 2; // Compressed Size if (block->compressed_size != LZMA_VLI_UNKNOWN) { @@ -114,6 +108,7 @@ lzma_block_header_encode(const lzma_block *block, uint8_t *out) if (block->filters == NULL || block->filters[0].id == LZMA_VLI_UNKNOWN) return LZMA_PROG_ERROR; + size_t filter_count = 0; do { // There can be a maximum of four filters. if (filter_count == LZMA_FILTERS_MAX) diff --git a/Utilities/cmliblzma/liblzma/common/block_util.c b/Utilities/cmliblzma/liblzma/common/block_util.c index 4cd34d1..00c7fe8 100644 --- a/Utilities/cmliblzma/liblzma/common/block_util.c +++ b/Utilities/cmliblzma/liblzma/common/block_util.c @@ -17,14 +17,11 @@ extern LZMA_API(lzma_ret) lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) { - uint32_t container_size; - lzma_vli compressed_size; - // Validate everything but Uncompressed Size and filters. if (lzma_block_unpadded_size(block) == 0) return LZMA_PROG_ERROR; - container_size = block->header_size + const uint32_t container_size = block->header_size + lzma_check_size(block->check); // Validate that Compressed Size will be greater than zero. @@ -34,7 +31,7 @@ lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) // Calculate what Compressed Size is supposed to be. // If Compressed Size was present in Block Header, // compare that the new value matches it. - compressed_size = unpadded_size - container_size; + const lzma_vli compressed_size = unpadded_size - container_size; if (block->compressed_size != LZMA_VLI_UNKNOWN && block->compressed_size != compressed_size) return LZMA_DATA_ERROR; @@ -48,15 +45,13 @@ lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) extern LZMA_API(lzma_vli) lzma_block_unpadded_size(const lzma_block *block) { - lzma_vli unpadded_size; - // Validate the values that we are interested in i.e. all but // Uncompressed Size and the filters. // // NOTE: This function is used for validation too, so it is // essential that these checks are always done even if // Compressed Size is unknown. - if (block == NULL || block->version != 0 + if (block == NULL || block->version > 1 || block->header_size < LZMA_BLOCK_HEADER_SIZE_MIN || block->header_size > LZMA_BLOCK_HEADER_SIZE_MAX || (block->header_size & 3) @@ -71,7 +66,7 @@ lzma_block_unpadded_size(const lzma_block *block) return LZMA_VLI_UNKNOWN; // Calculate Unpadded Size and validate it. - unpadded_size = block->compressed_size + const lzma_vli unpadded_size = block->compressed_size + block->header_size + lzma_check_size(block->check); diff --git a/Utilities/cmliblzma/liblzma/common/common.c b/Utilities/cmliblzma/liblzma/common/common.c index 2e723c8..57e3f8e 100644 --- a/Utilities/cmliblzma/liblzma/common/common.c +++ b/Utilities/cmliblzma/liblzma/common/common.c @@ -36,14 +36,14 @@ lzma_version_string(void) /////////////////////// extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) -lzma_alloc(size_t size, lzma_allocator *allocator) +lzma_alloc(size_t size, const lzma_allocator *allocator) { - void *ptr; - // Some malloc() variants return NULL if called with size == 0. if (size == 0) size = 1; + void *ptr; + if (allocator != NULL && allocator->alloc != NULL) ptr = allocator->alloc(allocator->opaque, 1, size); else @@ -53,8 +53,29 @@ lzma_alloc(size_t size, lzma_allocator *allocator) } +extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) +lzma_alloc_zero(size_t size, const lzma_allocator *allocator) +{ + // Some calloc() variants return NULL if called with size == 0. + if (size == 0) + size = 1; + + void *ptr; + + if (allocator != NULL && allocator->alloc != NULL) { + ptr = allocator->alloc(allocator->opaque, 1, size); + if (ptr != NULL) + memzero(ptr, size); + } else { + ptr = calloc(1, size); + } + + return ptr; +} + + extern void -lzma_free(void *ptr, lzma_allocator *allocator) +lzma_free(void *ptr, const lzma_allocator *allocator) { if (allocator != NULL && allocator->free != NULL) allocator->free(allocator->opaque, ptr); @@ -70,9 +91,9 @@ lzma_free(void *ptr, lzma_allocator *allocator) ////////// extern size_t -lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size) +lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) { const size_t in_avail = in_size - *in_pos; const size_t out_avail = out_size - *out_pos; @@ -88,7 +109,7 @@ lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, extern lzma_ret -lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_next_filter_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { lzma_next_coder_init(filters[0].init, next, allocator); @@ -99,7 +120,7 @@ lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_next_filter_update(lzma_next_coder *next, lzma_allocator *allocator, +lzma_next_filter_update(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *reversed_filters) { // Check that the application isn't trying to change the Filter ID. @@ -117,7 +138,7 @@ lzma_next_filter_update(lzma_next_coder *next, lzma_allocator *allocator, extern void -lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator) +lzma_next_end(lzma_next_coder *next, const lzma_allocator *allocator) { if (next->init != (uintptr_t)(NULL)) { // To avoid tiny end functions that simply call @@ -156,10 +177,8 @@ lzma_strm_init(lzma_stream *strm) strm->internal->next = LZMA_NEXT_CODER_INIT; } - strm->internal->supported_actions[LZMA_RUN] = false; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = false; - strm->internal->supported_actions[LZMA_FULL_FLUSH] = false; - strm->internal->supported_actions[LZMA_FINISH] = false; + memzero(strm->internal->supported_actions, + sizeof(strm->internal->supported_actions)); strm->internal->sequence = ISEQ_RUN; strm->internal->allow_buf_error = false; @@ -173,16 +192,12 @@ lzma_strm_init(lzma_stream *strm) extern LZMA_API(lzma_ret) lzma_code(lzma_stream *strm, lzma_action action) { - size_t in_pos = 0; - size_t out_pos = 0; - lzma_ret ret; - // Sanity checks if ((strm->next_in == NULL && strm->avail_in != 0) || (strm->next_out == NULL && strm->avail_out != 0) || strm->internal == NULL || strm->internal->next.code == NULL - || (unsigned int)(action) > LZMA_FINISH + || (unsigned int)(action) > LZMA_ACTION_MAX || !strm->internal->supported_actions[action]) return LZMA_PROG_ERROR; @@ -217,6 +232,10 @@ lzma_code(lzma_stream *strm, lzma_action action) case LZMA_FINISH: strm->internal->sequence = ISEQ_FINISH; break; + + case LZMA_FULL_BARRIER: + strm->internal->sequence = ISEQ_FULL_BARRIER; + break; } break; @@ -244,6 +263,13 @@ lzma_code(lzma_stream *strm, lzma_action action) break; + case ISEQ_FULL_BARRIER: + if (action != LZMA_FULL_BARRIER + || strm->internal->avail_in != strm->avail_in) + return LZMA_PROG_ERROR; + + break; + case ISEQ_END: return LZMA_STREAM_END; @@ -252,7 +278,9 @@ lzma_code(lzma_stream *strm, lzma_action action) return LZMA_PROG_ERROR; } - ret = strm->internal->next.code( + size_t in_pos = 0; + size_t out_pos = 0; + lzma_ret ret = strm->internal->next.code( strm->internal->next.coder, strm->allocator, strm->next_in, &in_pos, strm->avail_in, strm->next_out, &out_pos, strm->avail_out, action); @@ -267,7 +295,9 @@ lzma_code(lzma_stream *strm, lzma_action action) strm->internal->avail_in = strm->avail_in; - switch (ret) { + // Cast is needed to silence a warning about LZMA_TIMED_OUT, which + // isn't part of lzma_ret enumeration. + switch ((unsigned int)(ret)) { case LZMA_OK: // Don't return LZMA_BUF_ERROR when it happens the first time. // This is to avoid returning LZMA_BUF_ERROR when avail_out @@ -283,9 +313,16 @@ lzma_code(lzma_stream *strm, lzma_action action) } break; + case LZMA_TIMED_OUT: + strm->internal->allow_buf_error = false; + ret = LZMA_OK; + break; + case LZMA_STREAM_END: if (strm->internal->sequence == ISEQ_SYNC_FLUSH - || strm->internal->sequence == ISEQ_FULL_FLUSH) + || strm->internal->sequence == ISEQ_FULL_FLUSH + || strm->internal->sequence + == ISEQ_FULL_BARRIER) strm->internal->sequence = ISEQ_RUN; else strm->internal->sequence = ISEQ_END; @@ -325,6 +362,22 @@ lzma_end(lzma_stream *strm) } +extern LZMA_API(void) +lzma_get_progress(lzma_stream *strm, + uint64_t *progress_in, uint64_t *progress_out) +{ + if (strm->internal->next.get_progress != NULL) { + strm->internal->next.get_progress(strm->internal->next.coder, + progress_in, progress_out); + } else { + *progress_in = strm->total_in; + *progress_out = strm->total_out; + } + + return; +} + + extern LZMA_API(lzma_check) lzma_get_check(const lzma_stream *strm) { @@ -382,8 +435,10 @@ lzma_memlimit_set(lzma_stream *strm, uint64_t new_memlimit) || strm->internal->next.memconfig == NULL) return LZMA_PROG_ERROR; - if (new_memlimit != 0 && new_memlimit < LZMA_MEMUSAGE_BASE) - return LZMA_MEMLIMIT_ERROR; + // Zero is a special value that cannot be used as an actual limit. + // If 0 was specified, use 1 instead. + if (new_memlimit == 0) + new_memlimit = 1; return strm->internal->next.memconfig(strm->internal->next.coder, &memusage, &old_memlimit, new_memlimit); diff --git a/Utilities/cmliblzma/liblzma/common/common.h b/Utilities/cmliblzma/liblzma/common/common.h index a6a2818..dde3ae0 100644 --- a/Utilities/cmliblzma/liblzma/common/common.h +++ b/Utilities/cmliblzma/liblzma/common/common.h @@ -48,6 +48,13 @@ #define LZMA_BUFFER_SIZE 4096 +/// Maximum number of worker threads within one multithreaded component. +/// The limit exists solely to make it simpler to prevent integer overflows +/// when allocating structures etc. This should be big enough for now... +/// the code won't scale anywhere close to this number anyway. +#define LZMA_THREADS_MAX 16384 + + /// Starting value for memory usage estimates. Instead of calculating size /// of _every_ structure and taking into account malloc() overhead etc., we /// add a base size to all memory usage estimates. It's not very accurate @@ -65,12 +72,20 @@ ( LZMA_TELL_NO_CHECK \ | LZMA_TELL_UNSUPPORTED_CHECK \ | LZMA_TELL_ANY_CHECK \ + | LZMA_IGNORE_CHECK \ | LZMA_CONCATENATED ) -/// Type of encoder/decoder specific data; the actual structure is defined -/// differently in different coders. -typedef struct lzma_coder_s lzma_coder; +/// Largest valid lzma_action value as unsigned integer. +#define LZMA_ACTION_MAX ((unsigned int)(LZMA_FULL_BARRIER)) + + +/// Special return value (lzma_ret) to indicate that a timeout was reached +/// and lzma_code() must not return LZMA_BUF_ERROR. This is converted to +/// LZMA_OK in lzma_code(). This is not in the lzma_ret enumeration because +/// there's no need to have it in the public API. +#define LZMA_TIMED_OUT 32 + typedef struct lzma_next_coder_s lzma_next_coder; @@ -79,7 +94,7 @@ typedef struct lzma_filter_info_s lzma_filter_info; /// Type of a function used to initialize a filter encoder or decoder typedef lzma_ret (*lzma_init_function)( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); /// Type of a function to do some kind of coding work (filters, Stream, @@ -87,15 +102,15 @@ typedef lzma_ret (*lzma_init_function)( /// input and output buffers, but for simplicity they still use this same /// function prototype. typedef lzma_ret (*lzma_code_function)( - lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, + void *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action); /// Type of a function to free the memory allocated for the coder typedef void (*lzma_end_function)( - lzma_coder *coder, lzma_allocator *allocator); + void *coder, const lzma_allocator *allocator); /// Raw coder validates and converts an array of lzma_filter structures to @@ -118,7 +133,7 @@ struct lzma_filter_info_s { /// Hold data and function pointers of the next filter in the chain. struct lzma_next_coder_s { /// Pointer to coder-specific data - lzma_coder *coder; + void *coder; /// Filter ID. This is LZMA_VLI_UNKNOWN when this structure doesn't /// point to a filter coder. @@ -138,35 +153,41 @@ struct lzma_next_coder_s { /// lzma_next_coder.coder. lzma_end_function end; + /// Pointer to a function to get progress information. If this is NULL, + /// lzma_stream.total_in and .total_out are used instead. + void (*get_progress)(void *coder, + uint64_t *progress_in, uint64_t *progress_out); + /// Pointer to function to return the type of the integrity check. /// Most coders won't support this. - lzma_check (*get_check)(const lzma_coder *coder); + lzma_check (*get_check)(const void *coder); /// Pointer to function to get and/or change the memory usage limit. /// If new_memlimit == 0, the limit is not changed. - lzma_ret (*memconfig)(lzma_coder *coder, uint64_t *memusage, + lzma_ret (*memconfig)(void *coder, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit); /// Update the filter-specific options or the whole filter chain /// in the encoder. - lzma_ret (*update)(lzma_coder *coder, lzma_allocator *allocator, + lzma_ret (*update)(void *coder, const lzma_allocator *allocator, const lzma_filter *filters, const lzma_filter *reversed_filters); }; -/// Constant to initialize lzma_next_coder structure -static const lzma_next_coder LZMA_NEXT_CODER_INIT = - { - NULL, - LZMA_VLI_UNKNOWN, - (uintptr_t)(NULL), - NULL, - NULL, - NULL, - NULL, - NULL, - }; +/// Macro to initialize lzma_next_coder structure +#define LZMA_NEXT_CODER_INIT \ + (lzma_next_coder){ \ + .coder = NULL, \ + .init = (uintptr_t)(NULL), \ + .id = LZMA_VLI_UNKNOWN, \ + .code = NULL, \ + .end = NULL, \ + .get_progress = NULL, \ + .get_check = NULL, \ + .memconfig = NULL, \ + .update = NULL, \ + } /// Internal data for lzma_strm_init, lzma_code, and lzma_end. A pointer to @@ -184,6 +205,7 @@ struct lzma_internal_s { ISEQ_SYNC_FLUSH, ISEQ_FULL_FLUSH, ISEQ_FINISH, + ISEQ_FULL_BARRIER, ISEQ_END, ISEQ_ERROR, } sequence; @@ -194,7 +216,7 @@ struct lzma_internal_s { size_t avail_in; /// Indicates which lzma_action values are allowed by next.code. - bool supported_actions[4]; + bool supported_actions[LZMA_ACTION_MAX + 1]; /// If true, lzma_code will return LZMA_BUF_ERROR if no progress was /// made (no input consumed and no output produced by next.code). @@ -203,15 +225,21 @@ struct lzma_internal_s { /// Allocates memory -extern void *lzma_alloc(size_t size, lzma_allocator *allocator) +extern void *lzma_alloc(size_t size, const lzma_allocator *allocator) lzma_attribute((__malloc__)) lzma_attr_alloc_size(1); +/// Allocates memory and zeroes it (like calloc()). This can be faster +/// than lzma_alloc() + memzero() while being backward compatible with +/// custom allocators. +extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) + lzma_alloc_zero(size_t size, const lzma_allocator *allocator); + /// Frees memory -extern void lzma_free(void *ptr, lzma_allocator *allocator); +extern void lzma_free(void *ptr, const lzma_allocator *allocator); /// Allocates strm->internal if it is NULL, and initializes *strm and -/// strm->internal. This function is only called via lzma_next_strm_init2 macro. +/// strm->internal. This function is only called via lzma_next_strm_init macro. extern lzma_ret lzma_strm_init(lzma_stream *strm); /// Initializes the next filter in the chain, if any. This takes care of @@ -219,24 +247,26 @@ extern lzma_ret lzma_strm_init(lzma_stream *strm); /// than the filter being initialized now. This way the actual filter /// initialization functions don't need to use lzma_next_coder_init macro. extern lzma_ret lzma_next_filter_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); /// Update the next filter in the chain, if any. This checks that /// the application is not trying to change the Filter IDs. extern lzma_ret lzma_next_filter_update( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *reversed_filters); /// Frees the memory allocated for next->coder either using next->end or, /// if next->end is NULL, using lzma_free. -extern void lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator); +extern void lzma_next_end(lzma_next_coder *next, + const lzma_allocator *allocator); /// Copy as much data as possible from in[] to out[] and update *in_pos /// and *out_pos accordingly. Returns the number of bytes copied. -extern size_t lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size); +extern size_t lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); /// \brief Return if expression doesn't evaluate to LZMA_OK @@ -269,37 +299,15 @@ do { \ /// (The function being called will use lzma_next_coder_init()). If /// initialization fails, memory that wasn't freed by func() is freed /// along strm->internal. -#define lzma_next_strm_init1(func, strm, arg1) \ +#define lzma_next_strm_init(func, strm, ...) \ do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ -} while (0) - -#define lzma_next_strm_init2(func, strm, arg1, arg2) \ -do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1, arg2); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ -} while (0) - -#define lzma_next_strm_init3(func, strm, arg1, arg2, arg3) \ -do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1, arg2, arg3); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ + return_if_error(lzma_strm_init(strm)); \ + const lzma_ret ret_ = func(&(strm)->internal->next, \ + (strm)->allocator, __VA_ARGS__); \ + if (ret_ != LZMA_OK) { \ + lzma_end(strm); \ + return ret_; \ + } \ } while (0) #endif diff --git a/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c index c4be34c..48eb56f 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c @@ -15,8 +15,8 @@ extern LZMA_API(lzma_ret) lzma_easy_buffer_encode(uint32_t preset, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, - uint8_t *out, size_t *out_pos, size_t out_size) + const lzma_allocator *allocator, const uint8_t *in, + size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { lzma_options_easy opt_easy; if (lzma_easy_preset(&opt_easy, preset)) diff --git a/Utilities/cmliblzma/liblzma/common/easy_encoder.c b/Utilities/cmliblzma/liblzma/common/easy_encoder.c index d13ccd7..5cb492d 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/easy_encoder.c @@ -11,7 +11,6 @@ /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" -#include "stream_encoder.h" extern LZMA_API(lzma_ret) diff --git a/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c index 65665c1..6620986 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c @@ -14,30 +14,27 @@ extern LZMA_API(lzma_ret) -lzma_raw_buffer_decode(const lzma_filter *filters, lzma_allocator *allocator, +lzma_raw_buffer_decode( + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder next = LZMA_NEXT_CODER_INIT; - size_t in_start; - size_t out_start; - lzma_ret ret; - // Validate what isn't validated later in filter_common.c. if (in == NULL || in_pos == NULL || *in_pos > in_size || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; // Initialize the decoer. + lzma_next_coder next = LZMA_NEXT_CODER_INIT; return_if_error(lzma_raw_decoder_init(&next, allocator, filters)); // Store the positions so that we can restore them if something // goes wrong. - in_start = *in_pos; - out_start = *out_pos; + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; // Do the actual decoding and free decoder's memory. - ret = next.code(next.coder, allocator, in, in_pos, in_size, + lzma_ret ret = next.code(next.coder, allocator, in, in_pos, in_size, out, out_pos, out_size, LZMA_FINISH); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c index b23329f..dda18e3 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c @@ -14,29 +14,27 @@ extern LZMA_API(lzma_ret) -lzma_raw_buffer_encode(const lzma_filter *filters, lzma_allocator *allocator, - const uint8_t *in, size_t in_size, uint8_t *out, - size_t *out_pos, size_t out_size) +lzma_raw_buffer_encode( + const lzma_filter *filters, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder next = LZMA_NEXT_CODER_INIT; - size_t out_start; - size_t in_pos = 0; - lzma_ret ret; - // Validate what isn't validated later in filter_common.c. if ((in == NULL && in_size != 0) || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; // Initialize the encoder + lzma_next_coder next = LZMA_NEXT_CODER_INIT; return_if_error(lzma_raw_encoder_init(&next, allocator, filters)); // Store the output position so that we can restore it if // something goes wrong. - out_start = *out_pos; + const size_t out_start = *out_pos; // Do the actual encoding and free coder's memory. - ret = next.code(next.coder, allocator, in, &in_pos, in_size, + size_t in_pos = 0; + lzma_ret ret = next.code(next.coder, allocator, in, &in_pos, in_size, out, out_pos, out_size, LZMA_FINISH); lzma_next_end(&next, allocator); diff --git a/Utilities/cmliblzma/liblzma/common/filter_common.c b/Utilities/cmliblzma/liblzma/common/filter_common.c index d2b9e08..9ad5d5d 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_common.c +++ b/Utilities/cmliblzma/liblzma/common/filter_common.c @@ -36,101 +36,100 @@ static const struct { } features[] = { #if defined (HAVE_ENCODER_LZMA1) || defined(HAVE_DECODER_LZMA1) { - LZMA_FILTER_LZMA1, - sizeof(lzma_options_lzma), - false, - true, - true, + .id = LZMA_FILTER_LZMA1, + .options_size = sizeof(lzma_options_lzma), + .non_last_ok = false, + .last_ok = true, + .changes_size = true, }, #endif #if defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2) { - LZMA_FILTER_LZMA2, - sizeof(lzma_options_lzma), - false, - true, - true, + .id = LZMA_FILTER_LZMA2, + .options_size = sizeof(lzma_options_lzma), + .non_last_ok = false, + .last_ok = true, + .changes_size = true, }, #endif #if defined(HAVE_ENCODER_X86) || defined(HAVE_DECODER_X86) { - LZMA_FILTER_X86, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_X86, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_POWERPC) || defined(HAVE_DECODER_POWERPC) { - LZMA_FILTER_POWERPC, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_POWERPC, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_IA64) || defined(HAVE_DECODER_IA64) { - LZMA_FILTER_IA64, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_IA64, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_ARM) || defined(HAVE_DECODER_ARM) { - LZMA_FILTER_ARM, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_ARM, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_ARMTHUMB) || defined(HAVE_DECODER_ARMTHUMB) { - LZMA_FILTER_ARMTHUMB, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_ARMTHUMB, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_SPARC) || defined(HAVE_DECODER_SPARC) { - LZMA_FILTER_SPARC, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_SPARC, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) { - LZMA_FILTER_DELTA, - sizeof(lzma_options_delta), - true, - false, - false, + .id = LZMA_FILTER_DELTA, + .options_size = sizeof(lzma_options_delta), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif { - LZMA_VLI_UNKNOWN + .id = LZMA_VLI_UNKNOWN } }; extern LZMA_API(lzma_ret) lzma_filters_copy(const lzma_filter *src, lzma_filter *dest, - lzma_allocator *allocator) + const lzma_allocator *allocator) { - size_t i; - lzma_ret ret; - if (src == NULL || dest == NULL) return LZMA_PROG_ERROR; + lzma_ret ret; + size_t i; for (i = 0; src[i].id != LZMA_VLI_UNKNOWN; ++i) { // There must be a maximum of four filters plus // the array terminator. @@ -194,6 +193,10 @@ error: static lzma_ret validate_chain(const lzma_filter *filters, size_t *count) { + // There must be at least one filter. + if (filters == NULL || filters[0].id == LZMA_VLI_UNKNOWN) + return LZMA_PROG_ERROR; + // Number of non-last filters that may change the size of the data // significantly (that is, more than 1-2 % or so). size_t changes_size_count = 0; @@ -207,11 +210,6 @@ validate_chain(const lzma_filter *filters, size_t *count) bool last_ok = false; size_t i = 0; - - // There must be at least one filter. - if (filters == NULL || filters[0].id == LZMA_VLI_UNKNOWN) - return LZMA_PROG_ERROR; - do { size_t j; for (j = 0; filters[i].id != features[j].id; ++j) @@ -241,21 +239,18 @@ validate_chain(const lzma_filter *filters, size_t *count) extern lzma_ret -lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options, lzma_filter_find coder_find, bool is_encoder) { - lzma_filter_info filters[LZMA_FILTERS_MAX + 1]; - size_t count; - size_t i; - lzma_ret ret; - // Do some basic validation and get the number of filters. + size_t count; return_if_error(validate_chain(options, &count)); // Set the filter functions and copy the options pointer. + lzma_filter_info filters[LZMA_FILTERS_MAX + 1]; if (is_encoder) { - for (i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { // The order of the filters is reversed in the // encoder. It allows more efficient handling // of the uncompressed data. @@ -271,7 +266,7 @@ lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, filters[j].options = options[i].options; } } else { - for (i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { const lzma_filter_coder *const fc = coder_find(options[i].id); if (fc == NULL || fc->init == NULL) @@ -288,7 +283,7 @@ lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, filters[count].init = NULL; // Initialize the filters. - ret = lzma_next_filter_init(next, allocator, filters); + const lzma_ret ret = lzma_next_filter_init(next, allocator, filters); if (ret != LZMA_OK) lzma_next_end(next, allocator); @@ -300,9 +295,6 @@ extern uint64_t lzma_raw_coder_memusage(lzma_filter_find coder_find, const lzma_filter *filters) { - uint64_t total = 0; - size_t i = 0; - // The chain has to have at least one filter. { size_t tmp; @@ -310,6 +302,9 @@ lzma_raw_coder_memusage(lzma_filter_find coder_find, return UINT64_MAX; } + uint64_t total = 0; + size_t i = 0; + do { const lzma_filter_coder *const fc = coder_find(filters[i].id); diff --git a/Utilities/cmliblzma/liblzma/common/filter_common.h b/Utilities/cmliblzma/liblzma/common/filter_common.h index cd61fc0..42a26a2 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_common.h +++ b/Utilities/cmliblzma/liblzma/common/filter_common.h @@ -36,7 +36,7 @@ typedef const lzma_filter_coder *(*lzma_filter_find)(lzma_vli id); extern lzma_ret lzma_raw_coder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters, lzma_filter_find coder_find, bool is_encoder); diff --git a/Utilities/cmliblzma/liblzma/common/filter_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_decoder.c index cce2b30..c75b0a8 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_decoder.c @@ -35,7 +35,8 @@ typedef struct { /// \return - LZMA_OK: Properties decoded successfully. /// - LZMA_OPTIONS_ERROR: Unsupported properties /// - LZMA_MEM_ERROR: Memory allocation failed. - lzma_ret (*props_decode)(void **options, lzma_allocator *allocator, + lzma_ret (*props_decode)( + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); } lzma_filter_decoder; @@ -44,74 +45,74 @@ typedef struct { static const lzma_filter_decoder decoders[] = { #ifdef HAVE_DECODER_LZMA1 { - LZMA_FILTER_LZMA1, - &lzma_lzma_decoder_init, - &lzma_lzma_decoder_memusage, - &lzma_lzma_props_decode, + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_decoder_init, + .memusage = &lzma_lzma_decoder_memusage, + .props_decode = &lzma_lzma_props_decode, }, #endif #ifdef HAVE_DECODER_LZMA2 { - LZMA_FILTER_LZMA2, - &lzma_lzma2_decoder_init, - &lzma_lzma2_decoder_memusage, - &lzma_lzma2_props_decode, + .id = LZMA_FILTER_LZMA2, + .init = &lzma_lzma2_decoder_init, + .memusage = &lzma_lzma2_decoder_memusage, + .props_decode = &lzma_lzma2_props_decode, }, #endif #ifdef HAVE_DECODER_X86 { - LZMA_FILTER_X86, - &lzma_simple_x86_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_X86, + .init = &lzma_simple_x86_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_POWERPC { - LZMA_FILTER_POWERPC, - &lzma_simple_powerpc_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_POWERPC, + .init = &lzma_simple_powerpc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_IA64 { - LZMA_FILTER_IA64, - &lzma_simple_ia64_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_IA64, + .init = &lzma_simple_ia64_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_ARM { - LZMA_FILTER_ARM, - &lzma_simple_arm_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_ARM, + .init = &lzma_simple_arm_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_ARMTHUMB { - LZMA_FILTER_ARMTHUMB, - &lzma_simple_armthumb_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_ARMTHUMB, + .init = &lzma_simple_armthumb_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_SPARC { - LZMA_FILTER_SPARC, - &lzma_simple_sparc_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_SPARC, + .init = &lzma_simple_sparc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_DELTA { - LZMA_FILTER_DELTA, - &lzma_delta_decoder_init, - &lzma_delta_coder_memusage, - &lzma_delta_props_decode, + .id = LZMA_FILTER_DELTA, + .init = &lzma_delta_decoder_init, + .memusage = &lzma_delta_coder_memusage, + .props_decode = &lzma_delta_props_decode, }, #endif }; @@ -120,8 +121,7 @@ static const lzma_filter_decoder decoders[] = { static const lzma_filter_decoder * decoder_find(lzma_vli id) { - size_t i; - for (i = 0; i < ARRAY_SIZE(decoders); ++i) + for (size_t i = 0; i < ARRAY_SIZE(decoders); ++i) if (decoders[i].id == id) return decoders + i; @@ -137,7 +137,7 @@ lzma_filter_decoder_is_supported(lzma_vli id) extern lzma_ret -lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options) { return lzma_raw_coder_init(next, allocator, @@ -148,7 +148,7 @@ lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_raw_decoder(lzma_stream *strm, const lzma_filter *options) { - lzma_next_strm_init1(lzma_raw_decoder_init, strm, options); + lzma_next_strm_init(lzma_raw_decoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -166,14 +166,13 @@ lzma_raw_decoder_memusage(const lzma_filter *filters) extern LZMA_API(lzma_ret) -lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator, +lzma_properties_decode(lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - const lzma_filter_decoder *const fd = decoder_find(filter->id); - // Make it always NULL so that the caller can always safely free() it. filter->options = NULL; + const lzma_filter_decoder *const fd = decoder_find(filter->id); if (fd == NULL) return LZMA_OPTIONS_ERROR; diff --git a/Utilities/cmliblzma/liblzma/common/filter_decoder.h b/Utilities/cmliblzma/liblzma/common/filter_decoder.h index d5c68bd..a2e255f 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/filter_decoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_raw_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options); #endif diff --git a/Utilities/cmliblzma/liblzma/common/filter_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_encoder.c index 9fdb100..c5d8f39 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_encoder.c @@ -30,11 +30,11 @@ typedef struct { /// invalid, UINT64_MAX is returned. uint64_t (*memusage)(const void *options); - /// Calculates the minimum sane size for Blocks (or other types of - /// chunks) to which the input data can be split to make - /// multithreaded encoding possible. If this is NULL, it is assumed - /// that the encoder is fast enough with single thread. - lzma_vli (*chunk_size)(const void *options); + /// Calculates the recommended Uncompressed Size for .xz Blocks to + /// which the input data can be split to make multithreaded + /// encoding possible. If this is NULL, it is assumed that + /// the encoder is fast enough with single thread. + uint64_t (*block_size)(const void *options); /// Tells the size of the Filter Properties field. If options are /// invalid, UINT32_MAX is returned. If this is NULL, props_size_fixed @@ -56,101 +56,95 @@ typedef struct { static const lzma_filter_encoder encoders[] = { #ifdef HAVE_ENCODER_LZMA1 { - LZMA_FILTER_LZMA1, - &lzma_lzma_encoder_init, - &lzma_lzma_encoder_memusage, - NULL, // FIXME - NULL, - 5, - &lzma_lzma_props_encode, + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_encoder_init, + .memusage = &lzma_lzma_encoder_memusage, + .block_size = NULL, // FIXME + .props_size_get = NULL, + .props_size_fixed = 5, + .props_encode = &lzma_lzma_props_encode, }, #endif #ifdef HAVE_ENCODER_LZMA2 { - LZMA_FILTER_LZMA2, - &lzma_lzma2_encoder_init, - &lzma_lzma2_encoder_memusage, - NULL, // FIXME - NULL, - 1, - &lzma_lzma2_props_encode, + .id = LZMA_FILTER_LZMA2, + .init = &lzma_lzma2_encoder_init, + .memusage = &lzma_lzma2_encoder_memusage, + .block_size = &lzma_lzma2_block_size, // FIXME + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_lzma2_props_encode, }, #endif #ifdef HAVE_ENCODER_X86 { - LZMA_FILTER_X86, - &lzma_simple_x86_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_X86, + .init = &lzma_simple_x86_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_POWERPC { - LZMA_FILTER_POWERPC, - &lzma_simple_powerpc_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_POWERPC, + .init = &lzma_simple_powerpc_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_IA64 { - LZMA_FILTER_IA64, - &lzma_simple_ia64_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_IA64, + .init = &lzma_simple_ia64_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_ARM { - LZMA_FILTER_ARM, - &lzma_simple_arm_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_ARM, + .init = &lzma_simple_arm_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_ARMTHUMB { - LZMA_FILTER_ARMTHUMB, - &lzma_simple_armthumb_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_ARMTHUMB, + .init = &lzma_simple_armthumb_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_SPARC { - LZMA_FILTER_SPARC, - &lzma_simple_sparc_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_SPARC, + .init = &lzma_simple_sparc_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_DELTA { - LZMA_FILTER_DELTA, - &lzma_delta_encoder_init, - &lzma_delta_coder_memusage, - NULL, - NULL, - 1, - &lzma_delta_props_encode, + .id = LZMA_FILTER_DELTA, + .init = &lzma_delta_encoder_init, + .memusage = &lzma_delta_coder_memusage, + .block_size = NULL, + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_delta_props_encode, }, #endif }; @@ -159,8 +153,7 @@ static const lzma_filter_encoder encoders[] = { static const lzma_filter_encoder * encoder_find(lzma_vli id) { - size_t i; - for (i = 0; i < ARRAY_SIZE(encoders); ++i) + for (size_t i = 0; i < ARRAY_SIZE(encoders); ++i) if (encoders[i].id == id) return encoders + i; @@ -178,10 +171,6 @@ lzma_filter_encoder_is_supported(lzma_vli id) extern LZMA_API(lzma_ret) lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) { - size_t i; - size_t count = 1; - lzma_filter reversed_filters[LZMA_FILTERS_MAX + 1]; - if (strm->internal->next.update == NULL) return LZMA_PROG_ERROR; @@ -191,10 +180,12 @@ lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) // The actual filter chain in the encoder is reversed. Some things // still want the normal order chain, so we provide both. + size_t count = 1; while (filters[count].id != LZMA_VLI_UNKNOWN) ++count; - for (i = 0; i < count; ++i) + lzma_filter reversed_filters[LZMA_FILTERS_MAX + 1]; + for (size_t i = 0; i < count; ++i) reversed_filters[count - i - 1] = filters[i]; reversed_filters[count].id = LZMA_VLI_UNKNOWN; @@ -205,7 +196,7 @@ lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) extern lzma_ret -lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options) { return lzma_raw_coder_init(next, allocator, @@ -216,7 +207,7 @@ lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options) { - lzma_next_strm_init3(lzma_raw_coder_init, strm, options, + lzma_next_strm_init(lzma_raw_coder_init, strm, options, (lzma_filter_find)(&encoder_find), true); strm->internal->supported_actions[LZMA_RUN] = true; @@ -235,20 +226,19 @@ lzma_raw_encoder_memusage(const lzma_filter *filters) } -/* -extern LZMA_API(lzma_vli) -lzma_chunk_size(const lzma_filter *filters) +extern uint64_t +lzma_mt_block_size(const lzma_filter *filters) { - lzma_vli max = 0; + uint64_t max = 0; for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { const lzma_filter_encoder *const fe = encoder_find(filters[i].id); - if (fe->chunk_size != NULL) { - const lzma_vli size - = fe->chunk_size(filters[i].options); - if (size == LZMA_VLI_UNKNOWN) - return LZMA_VLI_UNKNOWN; + if (fe->block_size != NULL) { + const uint64_t size + = fe->block_size(filters[i].options); + if (size == 0) + return 0; if (size > max) max = size; @@ -257,7 +247,6 @@ lzma_chunk_size(const lzma_filter *filters) return max; } -*/ extern LZMA_API(lzma_ret) diff --git a/Utilities/cmliblzma/liblzma/common/filter_encoder.h b/Utilities/cmliblzma/liblzma/common/filter_encoder.h index 5bc137f..f1d5683 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/filter_encoder.h @@ -16,12 +16,12 @@ #include "common.h" -// FIXME: Might become a part of the public API once finished. -// extern lzma_vli lzma_chunk_size(const lzma_filter *filters); +// FIXME: Might become a part of the public API. +extern uint64_t lzma_mt_block_size(const lzma_filter *filters); extern lzma_ret lzma_raw_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c index aa2dbd5..ddfb085 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c @@ -15,12 +15,9 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { - lzma_vli props_size; - lzma_ret ret; - // Set the pointer to NULL so the caller can always safely free it. filter->options = NULL; @@ -32,6 +29,7 @@ lzma_filter_flags_decode( return LZMA_DATA_ERROR; // Size of Properties + lzma_vli props_size; return_if_error(lzma_vli_decode(&props_size, NULL, in, in_pos, in_size)); @@ -39,7 +37,7 @@ lzma_filter_flags_decode( if (in_size - *in_pos < props_size) return LZMA_DATA_ERROR; - ret = lzma_properties_decode( + const lzma_ret ret = lzma_properties_decode( filter, allocator, in + *in_pos, props_size); *in_pos += props_size; diff --git a/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c index 755c407..d110566 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c @@ -31,8 +31,6 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filter, uint8_t *out, size_t *out_pos, size_t out_size) { - uint32_t props_size; - // Filter ID if (filter->id >= LZMA_FILTER_RESERVED_START) return LZMA_PROG_ERROR; @@ -41,6 +39,7 @@ lzma_filter_flags_encode(const lzma_filter *filter, out, out_pos, out_size)); // Size of Properties + uint32_t props_size; return_if_error(lzma_properties_size(&props_size, filter)); return_if_error(lzma_vli_encode(props_size, NULL, out, out_pos, out_size)); diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder.h b/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c index 46a7aed..f468366 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file stream_encoder.h -/// \brief Encodes .xz Streams +/// \file hardware_cputhreads.c +/// \brief Get the number of CPU threads or cores // // Author: Lasse Collin // @@ -10,14 +10,13 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_STREAM_ENCODER_H -#define LZMA_STREAM_ENCODER_H - #include "common.h" +#include "tuklib_cpucores.h" -extern lzma_ret lzma_stream_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, - const lzma_filter *filters, lzma_check check); -#endif +extern LZMA_API(uint32_t) +lzma_cputhreads(void) +{ + return tuklib_cpucores(); +} diff --git a/Utilities/cmliblzma/liblzma/common/index.c b/Utilities/cmliblzma/liblzma/common/index.c index 26135d2..26e4e51 100644 --- a/Utilities/cmliblzma/liblzma/common/index.c +++ b/Utilities/cmliblzma/liblzma/common/index.c @@ -191,8 +191,8 @@ index_tree_init(index_tree *tree) /// Helper for index_tree_end() static void -index_tree_node_end(index_tree_node *node, lzma_allocator *allocator, - void (*free_func)(void *node, lzma_allocator *allocator)) +index_tree_node_end(index_tree_node *node, const lzma_allocator *allocator, + void (*free_func)(void *node, const lzma_allocator *allocator)) { // The tree won't ever be very huge, so recursion should be fine. // 20 levels in the tree is likely quite a lot already in practice. @@ -202,22 +202,21 @@ index_tree_node_end(index_tree_node *node, lzma_allocator *allocator, if (node->right != NULL) index_tree_node_end(node->right, allocator, free_func); - if (free_func != NULL) - free_func(node, allocator); - - lzma_free(node, allocator); + free_func(node, allocator); return; } -/// Free the meory allocated for a tree. If free_func is not NULL, -/// it is called on each node before freeing the node. This is used -/// to free the Record groups from each index_stream before freeing -/// the index_stream itself. +/// Free the memory allocated for a tree. Each node is freed using the +/// given free_func which is either &lzma_free or &index_stream_end. +/// The latter is used to free the Record groups from each index_stream +/// before freeing the index_stream itself. static void -index_tree_end(index_tree *tree, lzma_allocator *allocator, - void (*free_func)(void *node, lzma_allocator *allocator)) +index_tree_end(index_tree *tree, const lzma_allocator *allocator, + void (*free_func)(void *node, const lzma_allocator *allocator)) { + assert(free_func != NULL); + if (tree->root != NULL) index_tree_node_end(tree->root, allocator, free_func); @@ -230,7 +229,6 @@ index_tree_end(index_tree *tree, lzma_allocator *allocator, static void index_tree_append(index_tree *tree, index_tree_node *node) { - uint32_t up; node->parent = tree->rightmost; node->left = NULL; node->right = NULL; @@ -259,10 +257,8 @@ index_tree_append(index_tree *tree, index_tree_node *node) // and thus know the state of the tree just by looking at the node // count. From the node count we can calculate how many steps to go // up in the tree to find the rotation root. - up = tree->count ^ (UINT32_C(1) << bsr32(tree->count)); + uint32_t up = tree->count ^ (UINT32_C(1) << bsr32(tree->count)); if (up != 0) { - index_tree_node *pivot; - // Locate the root node for the rotation. up = ctz32(tree->count) + 2; do { @@ -270,7 +266,7 @@ index_tree_append(index_tree *tree, index_tree_node *node) } while (--up > 0); // Rotate left using node as the rotation root. - pivot = node->right; + index_tree_node *pivot = node->right; if (node->parent == NULL) { tree->root = pivot; @@ -342,8 +338,8 @@ index_tree_locate(const index_tree *tree, lzma_vli target) /// Allocate and initialize a new Stream using the given base offsets. static index_stream * index_stream_init(lzma_vli compressed_base, lzma_vli uncompressed_base, - lzma_vli stream_number, lzma_vli block_number_base, - lzma_allocator *allocator) + uint32_t stream_number, lzma_vli block_number_base, + const lzma_allocator *allocator) { index_stream *s = lzma_alloc(sizeof(index_stream), allocator); if (s == NULL) @@ -371,16 +367,17 @@ index_stream_init(lzma_vli compressed_base, lzma_vli uncompressed_base, /// Free the memory allocated for a Stream and its Record groups. static void -index_stream_end(void *node, lzma_allocator *allocator) +index_stream_end(void *node, const lzma_allocator *allocator) { index_stream *s = node; - index_tree_end(&s->groups, allocator, NULL); + index_tree_end(&s->groups, allocator, &lzma_free); + lzma_free(s, allocator); return; } static lzma_index * -index_init_plain(lzma_allocator *allocator) +index_init_plain(const lzma_allocator *allocator) { lzma_index *i = lzma_alloc(sizeof(lzma_index), allocator); if (i != NULL) { @@ -398,15 +395,13 @@ index_init_plain(lzma_allocator *allocator) extern LZMA_API(lzma_index *) -lzma_index_init(lzma_allocator *allocator) +lzma_index_init(const lzma_allocator *allocator) { - index_stream *s; - lzma_index *i = index_init_plain(allocator); if (i == NULL) return NULL; - s = index_stream_init(0, 0, 1, 0, allocator); + index_stream *s = index_stream_init(0, 0, 1, 0, allocator); if (s == NULL) { lzma_free(i, allocator); return NULL; @@ -419,7 +414,7 @@ lzma_index_init(lzma_allocator *allocator) extern LZMA_API(void) -lzma_index_end(lzma_index *i, lzma_allocator *allocator) +lzma_index_end(lzma_index *i, const lzma_allocator *allocator) { // NOTE: If you modify this function, check also the bottom // of lzma_index_cat(). @@ -605,8 +600,6 @@ lzma_index_padding_size(const lzma_index *i) extern LZMA_API(lzma_ret) lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) { - index_stream *s; - if (i == NULL || stream_flags == NULL) return LZMA_PROG_ERROR; @@ -614,7 +607,7 @@ lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) return_if_error(lzma_stream_flags_compare( stream_flags, stream_flags)); - s = (index_stream *)(i->streams.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); s->stream_flags = *stream_flags; return LZMA_OK; @@ -624,17 +617,14 @@ lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) extern LZMA_API(lzma_ret) lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding) { - index_stream *s; - lzma_vli old_stream_padding; - if (i == NULL || stream_padding > LZMA_VLI_MAX || (stream_padding & 3) != 0) return LZMA_PROG_ERROR; - s = (index_stream *)(i->streams.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); // Check that the new value won't make the file grow too big. - old_stream_padding = s->stream_padding; + const lzma_vli old_stream_padding = s->stream_padding; s->stream_padding = 0; if (lzma_index_file_size(i) + stream_padding > LZMA_VLI_MAX) { s->stream_padding = old_stream_padding; @@ -647,29 +637,23 @@ lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding) extern LZMA_API(lzma_ret) -lzma_index_append(lzma_index *i, lzma_allocator *allocator, +lzma_index_append(lzma_index *i, const lzma_allocator *allocator, lzma_vli unpadded_size, lzma_vli uncompressed_size) { - index_stream *s; - index_group *g; - lzma_vli compressed_base; - lzma_vli uncompressed_base; - uint32_t index_list_size_add; - // Validate. if (i == NULL || unpadded_size < UNPADDED_SIZE_MIN || unpadded_size > UNPADDED_SIZE_MAX || uncompressed_size > LZMA_VLI_MAX) return LZMA_PROG_ERROR; - s = (index_stream *)(i->streams.rightmost); - g = (index_group *)(s->groups.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); + index_group *g = (index_group *)(s->groups.rightmost); - compressed_base = g == NULL ? 0 + const lzma_vli compressed_base = g == NULL ? 0 : vli_ceil4(g->records[g->last].unpadded_sum); - uncompressed_base = g == NULL ? 0 + const lzma_vli uncompressed_base = g == NULL ? 0 : g->records[g->last].uncompressed_sum; - index_list_size_add = lzma_vli_size(unpadded_size) + const uint32_t index_list_size_add = lzma_vli_size(unpadded_size) + lzma_vli_size(uncompressed_size); // Check that the file size will stay within limits. @@ -780,10 +764,9 @@ index_cat_helper(const index_cat_info *info, index_stream *this) extern LZMA_API(lzma_ret) -lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, - lzma_allocator *allocator) +lzma_index_cat(lzma_index *restrict dest, lzma_index *restrict src, + const lzma_allocator *allocator) { - index_cat_info info; const lzma_vli dest_file_size = lzma_index_file_size(dest); // Check that we don't exceed the file size limits. @@ -813,12 +796,10 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, index_stream *s = (index_stream *)(dest->streams.rightmost); index_group *g = (index_group *)(s->groups.rightmost); if (g != NULL && g->last + 1 < g->allocated) { - index_group *newg; - assert(g->node.left == NULL); assert(g->node.right == NULL); - newg = lzma_alloc(sizeof(index_group) + index_group *newg = lzma_alloc(sizeof(index_group) + (g->last + 1) * sizeof(index_record), allocator); @@ -848,17 +829,21 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, s->groups.rightmost = &newg->node; lzma_free(g, allocator); + + // NOTE: newg isn't leaked here because + // newg == (void *)&newg->node. } } // Add all the Streams from src to dest. Update the base offsets // of each Stream from src. - info.uncompressed_size = dest->uncompressed_size; - info.file_size = dest_file_size; - info.stream_number_add = dest->streams.count; - info.block_number_add = dest->record_count; - info.streams = &dest->streams; - + const index_cat_info info = { + .uncompressed_size = dest->uncompressed_size, + .file_size = dest_file_size, + .stream_number_add = dest->streams.count, + .block_number_add = dest->record_count, + .streams = &dest->streams, + }; index_cat_helper(&info, (index_stream *)(src->streams.root)); // Update info about all the combined Streams. @@ -877,26 +862,18 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, /// Duplicate an index_stream. static index_stream * -index_dup_stream(const index_stream *src, lzma_allocator *allocator) +index_dup_stream(const index_stream *src, const lzma_allocator *allocator) { - index_stream *dest; - index_group *destg; - index_group *srcg; - size_t i = 0; - // Catch a somewhat theoretical integer overflow. if (src->record_count > PREALLOC_MAX) return NULL; // Allocate and initialize a new Stream. - dest = index_stream_init(src->node.compressed_base, + index_stream *dest = index_stream_init(src->node.compressed_base, src->node.uncompressed_base, src->number, src->block_number_base, allocator); - - // Return immediately if allocation failed or if there are - // no groups to duplicate. - if (dest == NULL || src->groups.leftmost == NULL) - return dest; + if (dest == NULL) + return NULL; // Copy the overall information. dest->record_count = src->record_count; @@ -904,10 +881,14 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) dest->stream_flags = src->stream_flags; dest->stream_padding = src->stream_padding; + // Return if there are no groups to duplicate. + if (src->groups.leftmost == NULL) + return dest; + // Allocate memory for the Records. We put all the Records into // a single group. It's simplest and also tends to make // lzma_index_locate() a little bit faster with very big Indexes. - destg = lzma_alloc(sizeof(index_group) + index_group *destg = lzma_alloc(sizeof(index_group) + src->record_count * sizeof(index_record), allocator); if (destg == NULL) { @@ -923,7 +904,8 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) destg->last = src->record_count - 1; // Go through all the groups in src and copy the Records into destg. - srcg = (index_group *)(src->groups.leftmost); + const index_group *srcg = (const index_group *)(src->groups.leftmost); + size_t i = 0; do { memcpy(destg->records + i, srcg->records, (srcg->last + 1) * sizeof(index_record)); @@ -941,11 +923,8 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) extern LZMA_API(lzma_index *) -lzma_index_dup(const lzma_index *src, lzma_allocator *allocator) +lzma_index_dup(const lzma_index *src, const lzma_allocator *allocator) { - index_stream *srcstream; - index_stream *deststream; - // Allocate the base structure (no initial Stream). lzma_index *dest = index_init_plain(allocator); if (dest == NULL) @@ -958,9 +937,11 @@ lzma_index_dup(const lzma_index *src, lzma_allocator *allocator) dest->index_list_size = src->index_list_size; // Copy the Streams and the groups in them. - srcstream = (index_stream *)(src->streams.leftmost); + const index_stream *srcstream + = (const index_stream *)(src->streams.leftmost); do { - deststream = index_dup_stream(srcstream, allocator); + index_stream *deststream = index_dup_stream( + srcstream, allocator); if (deststream == NULL) { lzma_index_end(dest, allocator); return NULL; @@ -1031,6 +1012,8 @@ iter_set_info(lzma_index_iter *iter) iter->internal[ITER_GROUP].p = NULL; } + // NOTE: lzma_index_iter.stream.number is lzma_vli but we use uint32_t + // internally. iter->stream.number = stream->number; iter->stream.block_count = stream->record_count; iter->stream.compressed_offset = stream->node.compressed_base; @@ -1119,19 +1102,14 @@ lzma_index_iter_rewind(lzma_index_iter *iter) extern LZMA_API(lzma_bool) lzma_index_iter_next(lzma_index_iter *iter, lzma_index_iter_mode mode) { - const lzma_index *i; - const index_stream *stream; - const index_group *group; - size_t record; - // Catch unsupported mode values. if ((unsigned int)(mode) > LZMA_INDEX_ITER_NONEMPTY_BLOCK) return true; - i = iter->internal[ITER_INDEX].p; - stream = iter->internal[ITER_STREAM].p; - group = NULL; - record = iter->internal[ITER_RECORD].s; + const lzma_index *i = iter->internal[ITER_INDEX].p; + const index_stream *stream = iter->internal[ITER_STREAM].p; + const index_group *group = NULL; + size_t record = iter->internal[ITER_RECORD].s; // If we are being asked for the next Stream, leave group to NULL // so that the rest of the this function thinks that this Stream @@ -1231,10 +1209,6 @@ again: extern LZMA_API(lzma_bool) lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) { - const index_stream *stream; - const index_group *group; - size_t left, right; - const lzma_index *i = iter->internal[ITER_INDEX].p; // If the target is past the end of the file, return immediately. @@ -1242,12 +1216,12 @@ lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) return true; // Locate the Stream containing the target offset. - stream = index_tree_locate(&i->streams, target); + const index_stream *stream = index_tree_locate(&i->streams, target); assert(stream != NULL); target -= stream->node.uncompressed_base; // Locate the group containing the target offset. - group = index_tree_locate(&stream->groups, target); + const index_group *group = index_tree_locate(&stream->groups, target); assert(group != NULL); // Use binary search to locate the exact Record. It is the first @@ -1255,8 +1229,8 @@ lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) // This is because we want the rightmost Record that fullfills the // search criterion. It is possible that there are empty Blocks; // we don't want to return them. - left = 0; - right = group->last; + size_t left = 0; + size_t right = group->last; while (left < right) { const size_t pos = left + (right - left) / 2; diff --git a/Utilities/cmliblzma/liblzma/common/index_decoder.c b/Utilities/cmliblzma/liblzma/common/index_decoder.c index 943cfd5..cc07a1b 100644 --- a/Utilities/cmliblzma/liblzma/common/index_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/index_decoder.c @@ -14,7 +14,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INDICATOR, SEQ_COUNT, @@ -50,18 +50,20 @@ struct lzma_coder_s { /// CRC32 of the List of Records field uint32_t crc32; -}; +} lzma_index_coder; static lzma_ret -index_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +index_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, - uint8_t *LZMA_RESTRICT out lzma_attribute((__unused__)), - size_t *LZMA_RESTRICT out_pos lzma_attribute((__unused__)), + uint8_t *restrict out lzma_attribute((__unused__)), + size_t *restrict out_pos lzma_attribute((__unused__)), size_t out_size lzma_attribute((__unused__)), lzma_action action lzma_attribute((__unused__))) { + lzma_index_coder *coder = coder_ptr; + // Similar optimization as in index_encoder.c const size_t in_start = *in_pos; lzma_ret ret = LZMA_OK; @@ -207,8 +209,9 @@ out: static void -index_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +index_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_index_coder *coder = coder_ptr; lzma_index_end(coder->index, allocator); lzma_free(coder, allocator); return; @@ -216,9 +219,11 @@ index_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +index_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_index_coder *coder = coder_ptr; + *memusage = lzma_index_memusage(1, coder->count); *old_memlimit = coder->memlimit; @@ -234,7 +239,7 @@ index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, static lzma_ret -index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, +index_decoder_reset(lzma_index_coder *coder, const lzma_allocator *allocator, lzma_index **i, uint64_t memlimit) { // Remember the pointer given by the application. We will set it @@ -251,7 +256,7 @@ index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, // Initialize the rest. coder->sequence = SEQ_INDICATOR; - coder->memlimit = memlimit; + coder->memlimit = my_max(1, memlimit); coder->count = 0; // Needs to be initialized due to _memconfig(). coder->pos = 0; coder->crc32 = 0; @@ -261,35 +266,37 @@ index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, static lzma_ret -index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_index **i, uint64_t memlimit) { lzma_next_coder_init(&index_decoder_init, next, allocator); - if (i == NULL || memlimit == 0) + if (i == NULL) return LZMA_PROG_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_index_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_index_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &index_decode; next->end = &index_decoder_end; next->memconfig = &index_decoder_memconfig; - next->coder->index = NULL; + coder->index = NULL; } else { - lzma_index_end(next->coder->index, allocator); + lzma_index_end(coder->index, allocator); } - return index_decoder_reset(next->coder, allocator, i, memlimit); + return index_decoder_reset(coder, allocator, i, memlimit); } extern LZMA_API(lzma_ret) lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) { - lzma_next_strm_init2(index_decoder_init, strm, i, memlimit); + lzma_next_strm_init(index_decoder_init, strm, i, memlimit); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -299,27 +306,25 @@ lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) extern LZMA_API(lzma_ret) -lzma_index_buffer_decode( - lzma_index **i, uint64_t *memlimit, lzma_allocator *allocator, +lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { - lzma_coder coder; - lzma_ret ret; - - // Store the input start position so that we can restore it in case - // of an error. - const size_t in_start = *in_pos; - // Sanity checks if (i == NULL || memlimit == NULL || in == NULL || in_pos == NULL || *in_pos > in_size) return LZMA_PROG_ERROR; // Initialize the decoder. + lzma_index_coder coder; return_if_error(index_decoder_reset(&coder, allocator, i, *memlimit)); + // Store the input start position so that we can restore it in case + // of an error. + const size_t in_start = *in_pos; + // Do the actual decoding. - ret = index_decode(&coder, allocator, in, in_pos, in_size, + lzma_ret ret = index_decode(&coder, allocator, in, in_pos, in_size, NULL, NULL, 0, LZMA_RUN); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/index_encoder.c b/Utilities/cmliblzma/liblzma/common/index_encoder.c index 194bf21..ac97d0c 100644 --- a/Utilities/cmliblzma/liblzma/common/index_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/index_encoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INDICATOR, SEQ_COUNT, @@ -37,19 +37,21 @@ struct lzma_coder_s { /// CRC32 of the List of Records field uint32_t crc32; -}; +} lzma_index_coder; static lzma_ret -index_encode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in lzma_attribute((__unused__)), - size_t *LZMA_RESTRICT in_pos lzma_attribute((__unused__)), +index_encode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in lzma_attribute((__unused__)), + size_t *restrict in_pos lzma_attribute((__unused__)), size_t in_size lzma_attribute((__unused__)), - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_action action lzma_attribute((__unused__))) { + lzma_index_coder *coder = coder_ptr; + // Position where to start calculating CRC32. The idea is that we // need to call lzma_crc32() only once per call to index_encode(). const size_t out_start = *out_pos; @@ -159,7 +161,7 @@ out: static void -index_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +index_encoder_end(void *coder, const lzma_allocator *allocator) { lzma_free(coder, allocator); return; @@ -167,7 +169,7 @@ index_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static void -index_encoder_reset(lzma_coder *coder, const lzma_index *i) +index_encoder_reset(lzma_index_coder *coder, const lzma_index *i) { lzma_index_iter_init(&coder->iter, i); @@ -181,7 +183,7 @@ index_encoder_reset(lzma_coder *coder, const lzma_index *i) extern lzma_ret -lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_index_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_index *i) { lzma_next_coder_init(&lzma_index_encoder_init, next, allocator); @@ -190,7 +192,7 @@ lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_PROG_ERROR; if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + next->coder = lzma_alloc(sizeof(lzma_index_coder), allocator); if (next->coder == NULL) return LZMA_MEM_ERROR; @@ -207,7 +209,7 @@ lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_index_encoder(lzma_stream *strm, const lzma_index *i) { - lzma_next_strm_init1(lzma_index_encoder_init, strm, i); + lzma_next_strm_init(lzma_index_encoder_init, strm, i); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -220,10 +222,6 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_coder coder; - size_t out_start; - lzma_ret ret; - // Validate the arguments. if (i == NULL || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; @@ -234,12 +232,13 @@ lzma_index_buffer_encode(const lzma_index *i, // The Index encoder needs just one small data structure so we can // allocate it on stack. + lzma_index_coder coder; index_encoder_reset(&coder, i); // Do the actual encoding. This should never fail, but store // the original *out_pos just in case. - out_start = *out_pos; - ret = index_encode(&coder, NULL, NULL, NULL, 0, + const size_t out_start = *out_pos; + lzma_ret ret = index_encode(&coder, NULL, NULL, NULL, 0, out, out_pos, out_size, LZMA_RUN); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/index_encoder.h b/Utilities/cmliblzma/liblzma/common/index_encoder.h index a13c94d..4d55cd1 100644 --- a/Utilities/cmliblzma/liblzma/common/index_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/index_encoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_index_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_index *i); + const lzma_allocator *allocator, const lzma_index *i); #endif diff --git a/Utilities/cmliblzma/liblzma/common/index_hash.c b/Utilities/cmliblzma/liblzma/common/index_hash.c index 0cf86b3..d7a0344 100644 --- a/Utilities/cmliblzma/liblzma/common/index_hash.c +++ b/Utilities/cmliblzma/liblzma/common/index_hash.c @@ -70,7 +70,8 @@ struct lzma_index_hash_s { extern LZMA_API(lzma_index_hash *) -lzma_index_hash_init(lzma_index_hash *index_hash, lzma_allocator *allocator) +lzma_index_hash_init(lzma_index_hash *index_hash, + const lzma_allocator *allocator) { if (index_hash == NULL) { index_hash = lzma_alloc(sizeof(lzma_index_hash), allocator); @@ -101,7 +102,8 @@ lzma_index_hash_init(lzma_index_hash *index_hash, lzma_allocator *allocator) extern LZMA_API(void) -lzma_index_hash_end(lzma_index_hash *index_hash, lzma_allocator *allocator) +lzma_index_hash_end(lzma_index_hash *index_hash, + const lzma_allocator *allocator) { lzma_free(index_hash, allocator); return; @@ -124,14 +126,13 @@ static lzma_ret hash_append(lzma_index_hash_info *info, lzma_vli unpadded_size, lzma_vli uncompressed_size) { - const lzma_vli sizes[2] = { unpadded_size, uncompressed_size }; - info->blocks_size += vli_ceil4(unpadded_size); info->uncompressed_size += uncompressed_size; info->index_list_size += lzma_vli_size(unpadded_size) + lzma_vli_size(uncompressed_size); ++info->count; + const lzma_vli sizes[2] = { unpadded_size, uncompressed_size }; lzma_check_update(&info->check, LZMA_CHECK_BEST, (const uint8_t *)(sizes), sizeof(sizes)); @@ -174,9 +175,6 @@ extern LZMA_API(lzma_ret) lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, size_t *in_pos, size_t in_size) { - size_t in_start; - lzma_ret ret; - // Catch zero input buffer here, because in contrast to Index encoder // and decoder functions, applications call this function directly // instead of via lzma_code(), which does the buffer checking. @@ -186,8 +184,8 @@ lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, // NOTE: This function has many similarities to index_encode() and // index_decode() functions found from index_encoder.c and // index_decoder.c. See the comments especially in index_encoder.c. - in_start = *in_pos; - ret = LZMA_OK; + const size_t in_start = *in_pos; + lzma_ret ret = LZMA_OK; while (*in_pos < in_size) switch (index_hash->sequence) { diff --git a/Utilities/cmliblzma/liblzma/common/memcmplen.h b/Utilities/cmliblzma/liblzma/common/memcmplen.h new file mode 100644 index 0000000..c1efc9e --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/memcmplen.h @@ -0,0 +1,175 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file memcmplen.h +/// \brief Optimized comparison of two buffers +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_MEMCMPLEN_H +#define LZMA_MEMCMPLEN_H + +#include "common.h" + +#ifdef HAVE_IMMINTRIN_H +# include <immintrin.h> +#endif + + +/// Find out how many equal bytes the two buffers have. +/// +/// \param buf1 First buffer +/// \param buf2 Second buffer +/// \param len How many bytes have already been compared and will +/// be assumed to match +/// \param limit How many bytes to compare at most, including the +/// already-compared bytes. This must be significantly +/// smaller than UINT32_MAX to avoid integer overflows. +/// Up to LZMA_MEMCMPLEN_EXTRA bytes may be read past +/// the specified limit from both buf1 and buf2. +/// +/// \return Number of equal bytes in the buffers is returned. +/// This is always at least len and at most limit. +/// +/// \note LZMA_MEMCMPLEN_EXTRA defines how many extra bytes may be read. +/// It's rounded up to 2^n. This extra amount needs to be +/// allocated in the buffers being used. It needs to be +/// initialized too to keep Valgrind quiet. +static inline uint32_t lzma_attribute((__always_inline__)) +lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2, + uint32_t len, uint32_t limit) +{ + assert(len <= limit); + assert(limit <= UINT32_MAX / 2); + +#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ + && ((TUKLIB_GNUC_REQ(3, 4) && defined(__x86_64__)) \ + || (defined(__INTEL_COMPILER) && defined(__x86_64__)) \ + || (defined(__INTEL_COMPILER) && defined(_M_X64)) \ + || (defined(_MSC_VER) && defined(_M_X64))) + // NOTE: This will use 64-bit unaligned access which + // TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit, but + // it's convenient here at least as long as it's x86-64 only. + // + // I keep this x86-64 only for now since that's where I know this + // to be a good method. This may be fine on other 64-bit CPUs too. + // On big endian one should use xor instead of subtraction and switch + // to __builtin_clzll(). +#define LZMA_MEMCMPLEN_EXTRA 8 + while (len < limit) { + const uint64_t x = *(const uint64_t *)(buf1 + len) + - *(const uint64_t *)(buf2 + len); + if (x != 0) { +# if defined(_M_X64) // MSVC or Intel C compiler on Windows + unsigned long tmp; + _BitScanForward64(&tmp, x); + len += (uint32_t)tmp >> 3; +# else // GCC, clang, or Intel C compiler + len += (uint32_t)__builtin_ctzll(x) >> 3; +# endif + return my_min(len, limit); + } + + len += 8; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ + && defined(HAVE__MM_MOVEMASK_EPI8) \ + && ((defined(__GNUC__) && defined(__SSE2_MATH__)) \ + || (defined(__INTEL_COMPILER) && defined(__SSE2__)) \ + || (defined(_MSC_VER) && defined(_M_IX86_FP) \ + && _M_IX86_FP >= 2)) + // NOTE: Like above, this will use 128-bit unaligned access which + // TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit. + // + // SSE2 version for 32-bit and 64-bit x86. On x86-64 the above + // version is sometimes significantly faster and sometimes + // slightly slower than this SSE2 version, so this SSE2 + // version isn't used on x86-64. +# define LZMA_MEMCMPLEN_EXTRA 16 + while (len < limit) { + const uint32_t x = 0xFFFF ^ _mm_movemask_epi8(_mm_cmpeq_epi8( + _mm_loadu_si128((const __m128i *)(buf1 + len)), + _mm_loadu_si128((const __m128i *)(buf2 + len)))); + + if (x != 0) { +# if defined(__INTEL_COMPILER) + len += _bit_scan_forward(x); +# elif defined(_MSC_VER) + unsigned long tmp; + _BitScanForward(&tmp, x); + len += tmp; +# else + len += __builtin_ctz(x); +# endif + return my_min(len, limit); + } + + len += 16; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && !defined(WORDS_BIGENDIAN) + // Generic 32-bit little endian method +# define LZMA_MEMCMPLEN_EXTRA 4 + while (len < limit) { + uint32_t x = *(const uint32_t *)(buf1 + len) + - *(const uint32_t *)(buf2 + len); + if (x != 0) { + if ((x & 0xFFFF) == 0) { + len += 2; + x >>= 16; + } + + if ((x & 0xFF) == 0) + ++len; + + return my_min(len, limit); + } + + len += 4; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && defined(WORDS_BIGENDIAN) + // Generic 32-bit big endian method +# define LZMA_MEMCMPLEN_EXTRA 4 + while (len < limit) { + uint32_t x = *(const uint32_t *)(buf1 + len) + ^ *(const uint32_t *)(buf2 + len); + if (x != 0) { + if ((x & 0xFFFF0000) == 0) { + len += 2; + x <<= 16; + } + + if ((x & 0xFF000000) == 0) + ++len; + + return my_min(len, limit); + } + + len += 4; + } + + return limit; + +#else + // Simple portable version that doesn't use unaligned access. +# define LZMA_MEMCMPLEN_EXTRA 0 + while (len < limit && buf1[len] == buf2[len]) + ++len; + + return len; +#endif +} + +#endif diff --git a/Utilities/cmliblzma/liblzma/common/outqueue.c b/Utilities/cmliblzma/liblzma/common/outqueue.c new file mode 100644 index 0000000..2dc8a38 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/outqueue.c @@ -0,0 +1,184 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file outqueue.c +/// \brief Output queue handling in multithreaded coding +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "outqueue.h" + + +/// This is to ease integer overflow checking: We may allocate up to +/// 2 * LZMA_THREADS_MAX buffers and we need some extra memory for other +/// data structures (that's the second /2). +#define BUF_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2) + + +static lzma_ret +get_options(uint64_t *bufs_alloc_size, uint32_t *bufs_count, + uint64_t buf_size_max, uint32_t threads) +{ + if (threads > LZMA_THREADS_MAX || buf_size_max > BUF_SIZE_MAX) + return LZMA_OPTIONS_ERROR; + + // The number of buffers is twice the number of threads. + // This wastes RAM but keeps the threads busy when buffers + // finish out of order. + // + // NOTE: If this is changed, update BUF_SIZE_MAX too. + *bufs_count = threads * 2; + *bufs_alloc_size = *bufs_count * buf_size_max; + + return LZMA_OK; +} + + +extern uint64_t +lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads) +{ + uint64_t bufs_alloc_size; + uint32_t bufs_count; + + if (get_options(&bufs_alloc_size, &bufs_count, buf_size_max, threads) + != LZMA_OK) + return UINT64_MAX; + + return sizeof(lzma_outq) + bufs_count * sizeof(lzma_outbuf) + + bufs_alloc_size; +} + + +extern lzma_ret +lzma_outq_init(lzma_outq *outq, const lzma_allocator *allocator, + uint64_t buf_size_max, uint32_t threads) +{ + uint64_t bufs_alloc_size; + uint32_t bufs_count; + + // Set bufs_count and bufs_alloc_size. + return_if_error(get_options(&bufs_alloc_size, &bufs_count, + buf_size_max, threads)); + + // Allocate memory if needed. + if (outq->buf_size_max != buf_size_max + || outq->bufs_allocated != bufs_count) { + lzma_outq_end(outq, allocator); + +#if SIZE_MAX < UINT64_MAX + if (bufs_alloc_size > SIZE_MAX) + return LZMA_MEM_ERROR; +#endif + + outq->bufs = lzma_alloc(bufs_count * sizeof(lzma_outbuf), + allocator); + outq->bufs_mem = lzma_alloc((size_t)(bufs_alloc_size), + allocator); + + if (outq->bufs == NULL || outq->bufs_mem == NULL) { + lzma_outq_end(outq, allocator); + return LZMA_MEM_ERROR; + } + } + + // Initialize the rest of the main structure. Initialization of + // outq->bufs[] is done when they are actually needed. + outq->buf_size_max = (size_t)(buf_size_max); + outq->bufs_allocated = bufs_count; + outq->bufs_pos = 0; + outq->bufs_used = 0; + outq->read_pos = 0; + + return LZMA_OK; +} + + +extern void +lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator) +{ + lzma_free(outq->bufs, allocator); + outq->bufs = NULL; + + lzma_free(outq->bufs_mem, allocator); + outq->bufs_mem = NULL; + + return; +} + + +extern lzma_outbuf * +lzma_outq_get_buf(lzma_outq *outq) +{ + // Caller must have checked it with lzma_outq_has_buf(). + assert(outq->bufs_used < outq->bufs_allocated); + + // Initialize the new buffer. + lzma_outbuf *buf = &outq->bufs[outq->bufs_pos]; + buf->buf = outq->bufs_mem + outq->bufs_pos * outq->buf_size_max; + buf->size = 0; + buf->finished = false; + + // Update the queue state. + if (++outq->bufs_pos == outq->bufs_allocated) + outq->bufs_pos = 0; + + ++outq->bufs_used; + + return buf; +} + + +extern bool +lzma_outq_is_readable(const lzma_outq *outq) +{ + uint32_t i = outq->bufs_pos - outq->bufs_used; + if (outq->bufs_pos < outq->bufs_used) + i += outq->bufs_allocated; + + return outq->bufs[i].finished; +} + + +extern lzma_ret +lzma_outq_read(lzma_outq *restrict outq, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_vli *restrict unpadded_size, + lzma_vli *restrict uncompressed_size) +{ + // There must be at least one buffer from which to read. + if (outq->bufs_used == 0) + return LZMA_OK; + + // Get the buffer. + uint32_t i = outq->bufs_pos - outq->bufs_used; + if (outq->bufs_pos < outq->bufs_used) + i += outq->bufs_allocated; + + lzma_outbuf *buf = &outq->bufs[i]; + + // If it isn't finished yet, we cannot read from it. + if (!buf->finished) + return LZMA_OK; + + // Copy from the buffer to output. + lzma_bufcpy(buf->buf, &outq->read_pos, buf->size, + out, out_pos, out_size); + + // Return if we didn't get all the data from the buffer. + if (outq->read_pos < buf->size) + return LZMA_OK; + + // The buffer was finished. Tell the caller its size information. + *unpadded_size = buf->unpadded_size; + *uncompressed_size = buf->uncompressed_size; + + // Free this buffer for further use. + --outq->bufs_used; + outq->read_pos = 0; + + return LZMA_STREAM_END; +} diff --git a/Utilities/cmliblzma/liblzma/common/outqueue.h b/Utilities/cmliblzma/liblzma/common/outqueue.h new file mode 100644 index 0000000..079634d --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/outqueue.h @@ -0,0 +1,156 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file outqueue.h +/// \brief Output queue handling in multithreaded coding +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +/// Output buffer for a single thread +typedef struct { + /// Pointer to the output buffer of lzma_outq.buf_size_max bytes + uint8_t *buf; + + /// Amount of data written to buf + size_t size; + + /// Additional size information + lzma_vli unpadded_size; + lzma_vli uncompressed_size; + + /// True when no more data will be written into this buffer. + /// + /// \note This is read by another thread and thus access + /// to this variable needs a mutex. + bool finished; + +} lzma_outbuf; + + +typedef struct { + /// Array of buffers that are used cyclically. + lzma_outbuf *bufs; + + /// Memory allocated for all the buffers + uint8_t *bufs_mem; + + /// Amount of buffer space available in each buffer + size_t buf_size_max; + + /// Number of buffers allocated + uint32_t bufs_allocated; + + /// Position in the bufs array. The next buffer to be taken + /// into use is bufs[bufs_pos]. + uint32_t bufs_pos; + + /// Number of buffers in use + uint32_t bufs_used; + + /// Position in the buffer in lzma_outq_read() + size_t read_pos; + +} lzma_outq; + + +/** + * \brief Calculate the memory usage of an output queue + * + * \return Approximate memory usage in bytes or UINT64_MAX on error. + */ +extern uint64_t lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads); + + +/// \brief Initialize an output queue +/// +/// \param outq Pointer to an output queue. Before calling +/// this function the first time, *outq should +/// have been zeroed with memzero() so that this +/// function knows that there are no previous +/// allocations to free. +/// \param allocator Pointer to allocator or NULL +/// \param buf_size_max Maximum amount of data that a single buffer +/// in the queue may need to store. +/// \param threads Number of buffers that may be in use +/// concurrently. Note that more than this number +/// of buffers will actually get allocated to +/// improve performance when buffers finish +/// out of order. +/// +/// \return - LZMA_OK +/// - LZMA_MEM_ERROR +/// +extern lzma_ret lzma_outq_init( + lzma_outq *outq, const lzma_allocator *allocator, + uint64_t buf_size_max, uint32_t threads); + + +/// \brief Free the memory associated with the output queue +extern void lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator); + + +/// \brief Get a new buffer +/// +/// lzma_outq_has_buf() must be used to check that there is a buffer +/// available before calling lzma_outq_get_buf(). +/// +extern lzma_outbuf *lzma_outq_get_buf(lzma_outq *outq); + + +/// \brief Test if there is data ready to be read +/// +/// Call to this function must be protected with the same mutex that +/// is used to protect lzma_outbuf.finished. +/// +extern bool lzma_outq_is_readable(const lzma_outq *outq); + + +/// \brief Read finished data +/// +/// \param outq Pointer to an output queue +/// \param out Beginning of the output buffer +/// \param out_pos The next byte will be written to +/// out[*out_pos]. +/// \param out_size Size of the out buffer; the first byte into +/// which no data is written to is out[out_size]. +/// \param unpadded_size Unpadded Size from the Block encoder +/// \param uncompressed_size Uncompressed Size from the Block encoder +/// +/// \return - LZMA: All OK. Either no data was available or the buffer +/// being read didn't become empty yet. +/// - LZMA_STREAM_END: The buffer being read was finished. +/// *unpadded_size and *uncompressed_size were set. +/// +/// \note This reads lzma_outbuf.finished variables and thus call +/// to this function needs to be protected with a mutex. +/// +extern lzma_ret lzma_outq_read(lzma_outq *restrict outq, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, lzma_vli *restrict unpadded_size, + lzma_vli *restrict uncompressed_size); + + +/// \brief Test if there is at least one buffer free +/// +/// This must be used before getting a new buffer with lzma_outq_get_buf(). +/// +static inline bool +lzma_outq_has_buf(const lzma_outq *outq) +{ + return outq->bufs_used < outq->bufs_allocated; +} + + +/// \brief Test if the queue is completely empty +static inline bool +lzma_outq_is_empty(const lzma_outq *outq) +{ + return outq->bufs_used == 0; +} diff --git a/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c index 9e2e1da..b9745b5 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c @@ -15,13 +15,10 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags, - lzma_allocator *allocator, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder stream_decoder = LZMA_NEXT_CODER_INIT; - lzma_ret ret; - // Sanity checks if (in_pos == NULL || (in == NULL && *in_pos != in_size) || *in_pos > in_size || out_pos == NULL @@ -36,7 +33,8 @@ lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags, // Initialize the Stream decoder. // TODO: We need something to tell the decoder that it can use the // output buffer as workspace, and thus save significant amount of RAM. - ret = lzma_stream_decoder_init( + lzma_next_coder stream_decoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_stream_decoder_init( &stream_decoder, allocator, *memlimit, flags); if (ret == LZMA_OK) { diff --git a/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c index 8bca87f..af49554 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c @@ -42,13 +42,10 @@ lzma_stream_buffer_bound(size_t uncompressed_size) extern LZMA_API(lzma_ret) lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos_ptr, size_t out_size) { - lzma_stream_flags stream_flags = { 0 }; - lzma_block block = { 0 }; - size_t out_pos; - // Sanity checks if (filters == NULL || (unsigned int)(check) > LZMA_CHECK_ID_MAX || (in == NULL && in_size != 0) || out == NULL @@ -65,7 +62,7 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, // Use a local copy. We update *out_pos_ptr only if everything // succeeds. - out_pos = *out_pos_ptr; + size_t out_pos = *out_pos_ptr; // Check that there's enough space for both Stream Header and // Stream Footer. @@ -77,7 +74,10 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, out_size -= LZMA_STREAM_HEADER_SIZE; // Encode the Stream Header. - stream_flags.check = check; + lzma_stream_flags stream_flags = { + .version = 0, + .check = check, + }; if (lzma_stream_header_encode(&stream_flags, out + out_pos) != LZMA_OK) @@ -86,8 +86,11 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, out_pos += LZMA_STREAM_HEADER_SIZE; // Encode a Block but only if there is at least one byte of input. - block.check = check; - block.filters = filters; + lzma_block block = { + .version = 0, + .check = check, + .filters = filters, + }; if (in_size > 0) return_if_error(lzma_block_buffer_encode(&block, allocator, @@ -95,8 +98,6 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, // Index { - lzma_ret ret; - // Create an Index. It will have one Record if there was // at least one byte of input to encode. Otherwise the // Index will be empty. @@ -104,7 +105,7 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, if (i == NULL) return LZMA_MEM_ERROR; - ret = LZMA_OK; + lzma_ret ret = LZMA_OK; if (in_size > 0) ret = lzma_index_append(i, allocator, diff --git a/Utilities/cmliblzma/liblzma/common/stream_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_decoder.c index 5e9a220..fdd8ff2 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_decoder.c @@ -14,7 +14,7 @@ #include "block_decoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_STREAM_HEADER, SEQ_BLOCK_HEADER, @@ -57,6 +57,10 @@ struct lzma_coder_s { /// If true, LZMA_GET_CHECK is returned after decoding Stream Header. bool tell_any_check; + /// If true, we will tell the Block decoder to skip calculating + /// and verifying the integrity check. + bool ignore_check; + /// If true, we will decode concatenated Streams that possibly have /// Stream Padding between or after them. LZMA_STREAM_END is returned /// once the application isn't giving us any new input, and we aren't @@ -76,11 +80,11 @@ struct lzma_coder_s { /// Buffer to hold Stream Header, Block Header, and Stream Footer. /// Block Header has biggest maximum size. uint8_t buffer[LZMA_BLOCK_HEADER_SIZE_MAX]; -}; +} lzma_stream_coder; static lzma_ret -stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator) +stream_decoder_reset(lzma_stream_coder *coder, const lzma_allocator *allocator) { // Initialize the Index hash used to verify the Index. coder->index_hash = lzma_index_hash_init(coder->index_hash, allocator); @@ -96,18 +100,18 @@ stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +stream_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_stream_coder *coder = coder_ptr; + // When decoding the actual Block, it may be able to produce more // output even if we don't give it any new input. while (true) switch (coder->sequence) { case SEQ_STREAM_HEADER: { - lzma_ret ret; - // Copy the Stream Header to the internal buffer. lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); @@ -119,7 +123,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, coder->pos = 0; // Decode the Stream Header. - ret = lzma_stream_header_decode( + const lzma_ret ret = lzma_stream_header_decode( &coder->stream_flags, coder->buffer); if (ret != LZMA_OK) return ret == LZMA_FORMAT_ERROR && !coder->first_stream @@ -156,11 +160,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_BLOCK_HEADER: { - lzma_filter filters[LZMA_FILTERS_MAX + 1]; - uint64_t memusage; - lzma_ret ret; - size_t i; - if (*in_pos >= in_size) return LZMA_OK; @@ -189,20 +188,28 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, coder->pos = 0; - // Version 0 is currently the only possible version. - coder->block_options.version = 0; + // Version 1 is needed to support the .ignore_check option. + coder->block_options.version = 1; // Set up a buffer to hold the filter chain. Block Header // decoder will initialize all members of this array so // we don't need to do it here. + lzma_filter filters[LZMA_FILTERS_MAX + 1]; coder->block_options.filters = filters; // Decode the Block Header. return_if_error(lzma_block_header_decode(&coder->block_options, allocator, coder->buffer)); + // If LZMA_IGNORE_CHECK was used, this flag needs to be set. + // It has to be set after lzma_block_header_decode() because + // it always resets this to false. + coder->block_options.ignore_check = coder->ignore_check; + // Check the memory usage limit. - memusage = lzma_raw_decoder_memusage(filters); + const uint64_t memusage = lzma_raw_decoder_memusage(filters); + lzma_ret ret; + if (memusage == UINT64_MAX) { // One or more unknown Filter IDs. ret = LZMA_OPTIONS_ERROR; @@ -228,7 +235,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Free the allocated filter options since they are needed // only to initialize the Block decoder. - for (i = 0; i < LZMA_FILTERS_MAX; ++i) + for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) lzma_free(filters[i].options, allocator); coder->block_options.filters = NULL; @@ -264,8 +271,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_INDEX: { - lzma_ret ret; - // If we don't have any input, don't call // lzma_index_hash_decode() since it would return // LZMA_BUF_ERROR, which we must not do here. @@ -274,7 +279,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Decode the Index and compare it to the hash calculated // from the sizes of the Blocks (if any). - ret = lzma_index_hash_decode(coder->index_hash, + const lzma_ret ret = lzma_index_hash_decode(coder->index_hash, in, in_pos, in_size); if (ret != LZMA_STREAM_END) return ret; @@ -285,9 +290,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_STREAM_FOOTER: { - lzma_stream_flags footer_flags; - lzma_ret ret; - // Copy the Stream Footer to the internal buffer. lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); @@ -301,7 +303,8 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Decode the Stream Footer. The decoder gives // LZMA_FORMAT_ERROR if the magic bytes don't match, // so convert that return code to LZMA_DATA_ERROR. - ret = lzma_stream_footer_decode( + lzma_stream_flags footer_flags; + const lzma_ret ret = lzma_stream_footer_decode( &footer_flags, coder->buffer); if (ret != LZMA_OK) return ret == LZMA_FORMAT_ERROR @@ -374,8 +377,9 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, static void -stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +stream_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_stream_coder *coder = coder_ptr; lzma_next_end(&coder->block_decoder, allocator); lzma_index_hash_end(coder->index_hash, allocator); lzma_free(coder, allocator); @@ -384,16 +388,19 @@ stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_check -stream_decoder_get_check(const lzma_coder *coder) +stream_decoder_get_check(const void *coder_ptr) { + const lzma_stream_coder *coder = coder_ptr; return coder->stream_flags.check; } static lzma_ret -stream_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +stream_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_stream_coder *coder = coder_ptr; + *memusage = coder->memusage; *old_memlimit = coder->memlimit; @@ -409,48 +416,49 @@ stream_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, extern lzma_ret -lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_stream_decoder_init( + lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, uint32_t flags) { lzma_next_coder_init(&lzma_stream_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; - if (flags & ~LZMA_SUPPORTED_FLAGS) return LZMA_OPTIONS_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_stream_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &stream_decode; next->end = &stream_decoder_end; next->get_check = &stream_decoder_get_check; next->memconfig = &stream_decoder_memconfig; - next->coder->block_decoder = LZMA_NEXT_CODER_INIT; - next->coder->index_hash = NULL; + coder->block_decoder = LZMA_NEXT_CODER_INIT; + coder->index_hash = NULL; } - next->coder->memlimit = memlimit; - next->coder->memusage = LZMA_MEMUSAGE_BASE; - next->coder->tell_no_check = (flags & LZMA_TELL_NO_CHECK) != 0; - next->coder->tell_unsupported_check + coder->memlimit = my_max(1, memlimit); + coder->memusage = LZMA_MEMUSAGE_BASE; + coder->tell_no_check = (flags & LZMA_TELL_NO_CHECK) != 0; + coder->tell_unsupported_check = (flags & LZMA_TELL_UNSUPPORTED_CHECK) != 0; - next->coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0; - next->coder->concatenated = (flags & LZMA_CONCATENATED) != 0; - next->coder->first_stream = true; + coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0; + coder->ignore_check = (flags & LZMA_IGNORE_CHECK) != 0; + coder->concatenated = (flags & LZMA_CONCATENATED) != 0; + coder->first_stream = true; - return stream_decoder_reset(next->coder, allocator); + return stream_decoder_reset(coder, allocator); } extern LZMA_API(lzma_ret) lzma_stream_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init2(lzma_stream_decoder_init, strm, memlimit, flags); + lzma_next_strm_init(lzma_stream_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/stream_decoder.h b/Utilities/cmliblzma/liblzma/common/stream_decoder.h index e54ac28..c13c6ba 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/stream_decoder.h @@ -15,7 +15,8 @@ #include "common.h" -extern lzma_ret lzma_stream_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, uint64_t memlimit, uint32_t flags); +extern lzma_ret lzma_stream_decoder_init( + lzma_next_coder *next, const lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags); #endif diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_encoder.c index 1ba45ac..858cba4 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_encoder.c @@ -10,12 +10,11 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_encoder.h" #include "block_encoder.h" #include "index_encoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_STREAM_HEADER, SEQ_BLOCK_INIT, @@ -26,7 +25,7 @@ struct lzma_coder_s { } sequence; /// True if Block encoder has been initialized by - /// lzma_stream_encoder_init() or stream_encoder_update() + /// stream_encoder_init() or stream_encoder_update() /// and thus doesn't need to be initialized in stream_encode(). bool block_encoder_is_initialized; @@ -56,11 +55,11 @@ struct lzma_coder_s { /// Buffer to hold Stream Header, Block Header, and Stream Footer. /// Block Header has biggest maximum size. uint8_t buffer[LZMA_BLOCK_HEADER_SIZE_MAX]; -}; +} lzma_stream_coder; static lzma_ret -block_encoder_init(lzma_coder *coder, lzma_allocator *allocator) +block_encoder_init(lzma_stream_coder *coder, const lzma_allocator *allocator) { // Prepare the Block options. Even though Block encoder doesn't need // compressed_size, uncompressed_size, and header_size to be @@ -79,11 +78,13 @@ block_encoder_init(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +stream_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_stream_coder *coder = coder_ptr; + // Main loop while (*out_pos < out_size) switch (coder->sequence) { @@ -126,7 +127,7 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } // Initialize the Block encoder unless it was already - // initialized by lzma_stream_encoder_init() or + // initialized by stream_encoder_init() or // stream_encoder_update(). if (!coder->block_encoder_is_initialized) return_if_error(block_encoder_init(coder, allocator)); @@ -147,13 +148,12 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_BLOCK_ENCODE: { - lzma_vli unpadded_size; - - static const lzma_action convert[4] = { + static const lzma_action convert[LZMA_ACTION_MAX + 1] = { LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FINISH, LZMA_FINISH, + LZMA_FINISH, }; const lzma_ret ret = coder->block_encoder.code( @@ -164,7 +164,7 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, return ret; // Add a new Index Record. - unpadded_size = lzma_block_unpadded_size( + const lzma_vli unpadded_size = lzma_block_unpadded_size( &coder->block_options); assert(unpadded_size != 0); return_if_error(lzma_index_append(coder->index, allocator, @@ -176,12 +176,6 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_INDEX_ENCODE: { - const lzma_stream_flags stream_flags = { - 0, - lzma_index_size(coder->index), - coder->block_options.check, - }; - // Call the Index encoder. It doesn't take any input, so // those pointers can be NULL. const lzma_ret ret = coder->index_encoder.code( @@ -192,6 +186,11 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, return ret; // Encode the Stream Footer into coder->buffer. + const lzma_stream_flags stream_flags = { + .version = 0, + .backward_size = lzma_index_size(coder->index), + .check = coder->block_options.check, + }; if (lzma_stream_footer_encode(&stream_flags, coder->buffer) != LZMA_OK) @@ -212,15 +211,15 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, static void -stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +stream_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { - size_t i; + lzma_stream_coder *coder = coder_ptr; lzma_next_end(&coder->block_encoder, allocator); lzma_next_end(&coder->index_encoder, allocator); lzma_index_end(coder->index, allocator); - for (i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) lzma_free(coder->filters[i].options, allocator); lzma_free(coder, allocator); @@ -229,22 +228,20 @@ stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +stream_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters, const lzma_filter *reversed_filters) { - size_t i; + lzma_stream_coder *coder = coder_ptr; if (coder->sequence <= SEQ_BLOCK_INIT) { - lzma_ret ret; - // There is no incomplete Block waiting to be finished, // thus we can change the whole filter chain. Start by // trying to initialize the Block encoder with the new // chain. This way we detect if the chain is valid. coder->block_encoder_is_initialized = false; coder->block_options.filters = (lzma_filter *)(filters); - ret = block_encoder_init(coder, allocator); + const lzma_ret ret = block_encoder_init(coder, allocator); coder->block_options.filters = coder->filters; if (ret != LZMA_OK) return ret; @@ -264,62 +261,66 @@ stream_encoder_update(lzma_coder *coder, lzma_allocator *allocator, } // Free the copy of the old chain and make a copy of the new chain. - for (i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) lzma_free(coder->filters[i].options, allocator); return lzma_filters_copy(filters, coder->filters, allocator); } -extern lzma_ret -lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +static lzma_ret +stream_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters, lzma_check check) { - lzma_stream_flags stream_flags = { 0, 0, check }; - - lzma_next_coder_init(&lzma_stream_encoder_init, next, allocator); + lzma_next_coder_init(&stream_encoder_init, next, allocator); if (filters == NULL) return LZMA_PROG_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_stream_coder *coder = next->coder; + + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &stream_encode; next->end = &stream_encoder_end; next->update = &stream_encoder_update; - next->coder->filters[0].id = LZMA_VLI_UNKNOWN; - next->coder->block_encoder = LZMA_NEXT_CODER_INIT; - next->coder->index_encoder = LZMA_NEXT_CODER_INIT; - next->coder->index = NULL; + coder->filters[0].id = LZMA_VLI_UNKNOWN; + coder->block_encoder = LZMA_NEXT_CODER_INIT; + coder->index_encoder = LZMA_NEXT_CODER_INIT; + coder->index = NULL; } // Basic initializations - next->coder->sequence = SEQ_STREAM_HEADER; - next->coder->block_options.version = 0; - next->coder->block_options.check = check; + coder->sequence = SEQ_STREAM_HEADER; + coder->block_options.version = 0; + coder->block_options.check = check; // Initialize the Index - lzma_index_end(next->coder->index, allocator); - next->coder->index = lzma_index_init(allocator); - if (next->coder->index == NULL) + lzma_index_end(coder->index, allocator); + coder->index = lzma_index_init(allocator); + if (coder->index == NULL) return LZMA_MEM_ERROR; // Encode the Stream Header + lzma_stream_flags stream_flags = { + .version = 0, + .check = check, + }; return_if_error(lzma_stream_header_encode( - &stream_flags, next->coder->buffer)); + &stream_flags, coder->buffer)); - next->coder->buffer_pos = 0; - next->coder->buffer_size = LZMA_STREAM_HEADER_SIZE; + coder->buffer_pos = 0; + coder->buffer_size = LZMA_STREAM_HEADER_SIZE; // Initialize the Block encoder. This way we detect unsupported // filter chains when initializing the Stream encoder instead of // giving an error after Stream Header has already written out. - return stream_encoder_update( - next->coder, allocator, filters, NULL); + return stream_encoder_update(coder, allocator, filters, NULL); } @@ -327,11 +328,12 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder(lzma_stream *strm, const lzma_filter *filters, lzma_check check) { - lzma_next_strm_init2(lzma_stream_encoder_init, strm, filters, check); + lzma_next_strm_init(stream_encoder_init, strm, filters, check); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; strm->internal->supported_actions[LZMA_FULL_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_BARRIER] = true; strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c b/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c new file mode 100644 index 0000000..2efe44c --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c @@ -0,0 +1,1143 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_encoder_mt.c +/// \brief Multithreaded .xz Stream encoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_encoder.h" +#include "easy_preset.h" +#include "block_encoder.h" +#include "block_buffer_encoder.h" +#include "index_encoder.h" +#include "outqueue.h" + + +/// Maximum supported block size. This makes it simpler to prevent integer +/// overflows if we are given unusually large block size. +#define BLOCK_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX) + + +typedef enum { + /// Waiting for work. + THR_IDLE, + + /// Encoding is in progress. + THR_RUN, + + /// Encoding is in progress but no more input data will + /// be read. + THR_FINISH, + + /// The main thread wants the thread to stop whatever it was doing + /// but not exit. + THR_STOP, + + /// The main thread wants the thread to exit. We could use + /// cancellation but since there's stopped anyway, this is lazier. + THR_EXIT, + +} worker_state; + +typedef struct lzma_stream_coder_s lzma_stream_coder; + +typedef struct worker_thread_s worker_thread; +struct worker_thread_s { + worker_state state; + + /// Input buffer of coder->block_size bytes. The main thread will + /// put new input into this and update in_size accordingly. Once + /// no more input is coming, state will be set to THR_FINISH. + uint8_t *in; + + /// Amount of data available in the input buffer. This is modified + /// only by the main thread. + size_t in_size; + + /// Output buffer for this thread. This is set by the main + /// thread every time a new Block is started with this thread + /// structure. + lzma_outbuf *outbuf; + + /// Pointer to the main structure is needed when putting this + /// thread back to the stack of free threads. + lzma_stream_coder *coder; + + /// The allocator is set by the main thread. Since a copy of the + /// pointer is kept here, the application must not change the + /// allocator before calling lzma_end(). + const lzma_allocator *allocator; + + /// Amount of uncompressed data that has already been compressed. + uint64_t progress_in; + + /// Amount of compressed data that is ready. + uint64_t progress_out; + + /// Block encoder + lzma_next_coder block_encoder; + + /// Compression options for this Block + lzma_block block_options; + + /// Next structure in the stack of free worker threads. + worker_thread *next; + + mythread_mutex mutex; + mythread_cond cond; + + /// The ID of this thread is used to join the thread + /// when it's not needed anymore. + mythread thread_id; +}; + + +struct lzma_stream_coder_s { + enum { + SEQ_STREAM_HEADER, + SEQ_BLOCK, + SEQ_INDEX, + SEQ_STREAM_FOOTER, + } sequence; + + /// Start a new Block every block_size bytes of input unless + /// LZMA_FULL_FLUSH or LZMA_FULL_BARRIER is used earlier. + size_t block_size; + + /// The filter chain currently in use + lzma_filter filters[LZMA_FILTERS_MAX + 1]; + + + /// Index to hold sizes of the Blocks + lzma_index *index; + + /// Index encoder + lzma_next_coder index_encoder; + + + /// Stream Flags for encoding the Stream Header and Stream Footer. + lzma_stream_flags stream_flags; + + /// Buffer to hold Stream Header and Stream Footer. + uint8_t header[LZMA_STREAM_HEADER_SIZE]; + + /// Read position in header[] + size_t header_pos; + + + /// Output buffer queue for compressed data + lzma_outq outq; + + + /// Maximum wait time if cannot use all the input and cannot + /// fill the output buffer. This is in milliseconds. + uint32_t timeout; + + + /// Error code from a worker thread + lzma_ret thread_error; + + /// Array of allocated thread-specific structures + worker_thread *threads; + + /// Number of structures in "threads" above. This is also the + /// number of threads that will be created at maximum. + uint32_t threads_max; + + /// Number of thread structures that have been initialized, and + /// thus the number of worker threads actually created so far. + uint32_t threads_initialized; + + /// Stack of free threads. When a thread finishes, it puts itself + /// back into this stack. This starts as empty because threads + /// are created only when actually needed. + worker_thread *threads_free; + + /// The most recent worker thread to which the main thread writes + /// the new input from the application. + worker_thread *thr; + + + /// Amount of uncompressed data in Blocks that have already + /// been finished. + uint64_t progress_in; + + /// Amount of compressed data in Stream Header + Blocks that + /// have already been finished. + uint64_t progress_out; + + + mythread_mutex mutex; + mythread_cond cond; +}; + + +/// Tell the main thread that something has gone wrong. +static void +worker_error(worker_thread *thr, lzma_ret ret) +{ + assert(ret != LZMA_OK); + assert(ret != LZMA_STREAM_END); + + mythread_sync(thr->coder->mutex) { + if (thr->coder->thread_error == LZMA_OK) + thr->coder->thread_error = ret; + + mythread_cond_signal(&thr->coder->cond); + } + + return; +} + + +static worker_state +worker_encode(worker_thread *thr, worker_state state) +{ + assert(thr->progress_in == 0); + assert(thr->progress_out == 0); + + // Set the Block options. + thr->block_options = (lzma_block){ + .version = 0, + .check = thr->coder->stream_flags.check, + .compressed_size = thr->coder->outq.buf_size_max, + .uncompressed_size = thr->coder->block_size, + + // TODO: To allow changing the filter chain, the filters + // array must be copied to each worker_thread. + .filters = thr->coder->filters, + }; + + // Calculate maximum size of the Block Header. This amount is + // reserved in the beginning of the buffer so that Block Header + // along with Compressed Size and Uncompressed Size can be + // written there. + lzma_ret ret = lzma_block_header_size(&thr->block_options); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + // Initialize the Block encoder. + ret = lzma_block_encoder_init(&thr->block_encoder, + thr->allocator, &thr->block_options); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + size_t in_pos = 0; + size_t in_size = 0; + + thr->outbuf->size = thr->block_options.header_size; + const size_t out_size = thr->coder->outq.buf_size_max; + + do { + mythread_sync(thr->mutex) { + // Store in_pos and out_pos into *thr so that + // an application may read them via + // lzma_get_progress() to get progress information. + // + // NOTE: These aren't updated when the encoding + // finishes. Instead, the final values are taken + // later from thr->outbuf. + thr->progress_in = in_pos; + thr->progress_out = thr->outbuf->size; + + while (in_size == thr->in_size + && thr->state == THR_RUN) + mythread_cond_wait(&thr->cond, &thr->mutex); + + state = thr->state; + in_size = thr->in_size; + } + + // Return if we were asked to stop or exit. + if (state >= THR_STOP) + return state; + + lzma_action action = state == THR_FINISH + ? LZMA_FINISH : LZMA_RUN; + + // Limit the amount of input given to the Block encoder + // at once. This way this thread can react fairly quickly + // if the main thread wants us to stop or exit. + static const size_t in_chunk_max = 16384; + size_t in_limit = in_size; + if (in_size - in_pos > in_chunk_max) { + in_limit = in_pos + in_chunk_max; + action = LZMA_RUN; + } + + ret = thr->block_encoder.code( + thr->block_encoder.coder, thr->allocator, + thr->in, &in_pos, in_limit, thr->outbuf->buf, + &thr->outbuf->size, out_size, action); + } while (ret == LZMA_OK && thr->outbuf->size < out_size); + + switch (ret) { + case LZMA_STREAM_END: + assert(state == THR_FINISH); + + // Encode the Block Header. By doing it after + // the compression, we can store the Compressed Size + // and Uncompressed Size fields. + ret = lzma_block_header_encode(&thr->block_options, + thr->outbuf->buf); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + break; + + case LZMA_OK: + // The data was incompressible. Encode it using uncompressed + // LZMA2 chunks. + // + // First wait that we have gotten all the input. + mythread_sync(thr->mutex) { + while (thr->state == THR_RUN) + mythread_cond_wait(&thr->cond, &thr->mutex); + + state = thr->state; + in_size = thr->in_size; + } + + if (state >= THR_STOP) + return state; + + // Do the encoding. This takes care of the Block Header too. + thr->outbuf->size = 0; + ret = lzma_block_uncomp_encode(&thr->block_options, + thr->in, in_size, thr->outbuf->buf, + &thr->outbuf->size, out_size); + + // It shouldn't fail. + if (ret != LZMA_OK) { + worker_error(thr, LZMA_PROG_ERROR); + return THR_STOP; + } + + break; + + default: + worker_error(thr, ret); + return THR_STOP; + } + + // Set the size information that will be read by the main thread + // to write the Index field. + thr->outbuf->unpadded_size + = lzma_block_unpadded_size(&thr->block_options); + assert(thr->outbuf->unpadded_size != 0); + thr->outbuf->uncompressed_size = thr->block_options.uncompressed_size; + + return THR_FINISH; +} + + +static MYTHREAD_RET_TYPE +worker_start(void *thr_ptr) +{ + worker_thread *thr = thr_ptr; + worker_state state = THR_IDLE; // Init to silence a warning + + while (true) { + // Wait for work. + mythread_sync(thr->mutex) { + while (true) { + // The thread is already idle so if we are + // requested to stop, just set the state. + if (thr->state == THR_STOP) { + thr->state = THR_IDLE; + mythread_cond_signal(&thr->cond); + } + + state = thr->state; + if (state != THR_IDLE) + break; + + mythread_cond_wait(&thr->cond, &thr->mutex); + } + } + + assert(state != THR_IDLE); + assert(state != THR_STOP); + + if (state <= THR_FINISH) + state = worker_encode(thr, state); + + if (state == THR_EXIT) + break; + + // Mark the thread as idle unless the main thread has + // told us to exit. Signal is needed for the case + // where the main thread is waiting for the threads to stop. + mythread_sync(thr->mutex) { + if (thr->state != THR_EXIT) { + thr->state = THR_IDLE; + mythread_cond_signal(&thr->cond); + } + } + + mythread_sync(thr->coder->mutex) { + // Mark the output buffer as finished if + // no errors occurred. + thr->outbuf->finished = state == THR_FINISH; + + // Update the main progress info. + thr->coder->progress_in + += thr->outbuf->uncompressed_size; + thr->coder->progress_out += thr->outbuf->size; + thr->progress_in = 0; + thr->progress_out = 0; + + // Return this thread to the stack of free threads. + thr->next = thr->coder->threads_free; + thr->coder->threads_free = thr; + + mythread_cond_signal(&thr->coder->cond); + } + } + + // Exiting, free the resources. + mythread_mutex_destroy(&thr->mutex); + mythread_cond_destroy(&thr->cond); + + lzma_next_end(&thr->block_encoder, thr->allocator); + lzma_free(thr->in, thr->allocator); + return MYTHREAD_RET_VALUE; +} + + +/// Make the threads stop but not exit. Optionally wait for them to stop. +static void +threads_stop(lzma_stream_coder *coder, bool wait_for_threads) +{ + // Tell the threads to stop. + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + coder->threads[i].state = THR_STOP; + mythread_cond_signal(&coder->threads[i].cond); + } + } + + if (!wait_for_threads) + return; + + // Wait for the threads to settle in the idle state. + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + while (coder->threads[i].state != THR_IDLE) + mythread_cond_wait(&coder->threads[i].cond, + &coder->threads[i].mutex); + } + } + + return; +} + + +/// Stop the threads and free the resources associated with them. +/// Wait until the threads have exited. +static void +threads_end(lzma_stream_coder *coder, const lzma_allocator *allocator) +{ + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + coder->threads[i].state = THR_EXIT; + mythread_cond_signal(&coder->threads[i].cond); + } + } + + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + int ret = mythread_join(coder->threads[i].thread_id); + assert(ret == 0); + (void)ret; + } + + lzma_free(coder->threads, allocator); + return; +} + + +/// Initialize a new worker_thread structure and create a new thread. +static lzma_ret +initialize_new_thread(lzma_stream_coder *coder, + const lzma_allocator *allocator) +{ + worker_thread *thr = &coder->threads[coder->threads_initialized]; + + thr->in = lzma_alloc(coder->block_size, allocator); + if (thr->in == NULL) + return LZMA_MEM_ERROR; + + if (mythread_mutex_init(&thr->mutex)) + goto error_mutex; + + if (mythread_cond_init(&thr->cond)) + goto error_cond; + + thr->state = THR_IDLE; + thr->allocator = allocator; + thr->coder = coder; + thr->progress_in = 0; + thr->progress_out = 0; + thr->block_encoder = LZMA_NEXT_CODER_INIT; + + if (mythread_create(&thr->thread_id, &worker_start, thr)) + goto error_thread; + + ++coder->threads_initialized; + coder->thr = thr; + + return LZMA_OK; + +error_thread: + mythread_cond_destroy(&thr->cond); + +error_cond: + mythread_mutex_destroy(&thr->mutex); + +error_mutex: + lzma_free(thr->in, allocator); + return LZMA_MEM_ERROR; +} + + +static lzma_ret +get_thread(lzma_stream_coder *coder, const lzma_allocator *allocator) +{ + // If there are no free output subqueues, there is no + // point to try getting a thread. + if (!lzma_outq_has_buf(&coder->outq)) + return LZMA_OK; + + // If there is a free structure on the stack, use it. + mythread_sync(coder->mutex) { + if (coder->threads_free != NULL) { + coder->thr = coder->threads_free; + coder->threads_free = coder->threads_free->next; + } + } + + if (coder->thr == NULL) { + // If there are no uninitialized structures left, return. + if (coder->threads_initialized == coder->threads_max) + return LZMA_OK; + + // Initialize a new thread. + return_if_error(initialize_new_thread(coder, allocator)); + } + + // Reset the parts of the thread state that have to be done + // in the main thread. + mythread_sync(coder->thr->mutex) { + coder->thr->state = THR_RUN; + coder->thr->in_size = 0; + coder->thr->outbuf = lzma_outq_get_buf(&coder->outq); + mythread_cond_signal(&coder->thr->cond); + } + + return LZMA_OK; +} + + +static lzma_ret +stream_encode_in(lzma_stream_coder *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, lzma_action action) +{ + while (*in_pos < in_size + || (coder->thr != NULL && action != LZMA_RUN)) { + if (coder->thr == NULL) { + // Get a new thread. + const lzma_ret ret = get_thread(coder, allocator); + if (coder->thr == NULL) + return ret; + } + + // Copy the input data to thread's buffer. + size_t thr_in_size = coder->thr->in_size; + lzma_bufcpy(in, in_pos, in_size, coder->thr->in, + &thr_in_size, coder->block_size); + + // Tell the Block encoder to finish if + // - it has got block_size bytes of input; or + // - all input was used and LZMA_FINISH, LZMA_FULL_FLUSH, + // or LZMA_FULL_BARRIER was used. + // + // TODO: LZMA_SYNC_FLUSH and LZMA_SYNC_BARRIER. + const bool finish = thr_in_size == coder->block_size + || (*in_pos == in_size && action != LZMA_RUN); + + bool block_error = false; + + mythread_sync(coder->thr->mutex) { + if (coder->thr->state == THR_IDLE) { + // Something has gone wrong with the Block + // encoder. It has set coder->thread_error + // which we will read a few lines later. + block_error = true; + } else { + // Tell the Block encoder its new amount + // of input and update the state if needed. + coder->thr->in_size = thr_in_size; + + if (finish) + coder->thr->state = THR_FINISH; + + mythread_cond_signal(&coder->thr->cond); + } + } + + if (block_error) { + lzma_ret ret; + + mythread_sync(coder->mutex) { + ret = coder->thread_error; + } + + return ret; + } + + if (finish) + coder->thr = NULL; + } + + return LZMA_OK; +} + + +/// Wait until more input can be consumed, more output can be read, or +/// an optional timeout is reached. +static bool +wait_for_work(lzma_stream_coder *coder, mythread_condtime *wait_abs, + bool *has_blocked, bool has_input) +{ + if (coder->timeout != 0 && !*has_blocked) { + // Every time when stream_encode_mt() is called via + // lzma_code(), *has_blocked starts as false. We set it + // to true here and calculate the absolute time when + // we must return if there's nothing to do. + // + // The idea of *has_blocked is to avoid unneeded calls + // to mythread_condtime_set(), which may do a syscall + // depending on the operating system. + *has_blocked = true; + mythread_condtime_set(wait_abs, &coder->cond, coder->timeout); + } + + bool timed_out = false; + + mythread_sync(coder->mutex) { + // There are four things that we wait. If one of them + // becomes possible, we return. + // - If there is input left, we need to get a free + // worker thread and an output buffer for it. + // - Data ready to be read from the output queue. + // - A worker thread indicates an error. + // - Time out occurs. + while ((!has_input || coder->threads_free == NULL + || !lzma_outq_has_buf(&coder->outq)) + && !lzma_outq_is_readable(&coder->outq) + && coder->thread_error == LZMA_OK + && !timed_out) { + if (coder->timeout != 0) + timed_out = mythread_cond_timedwait( + &coder->cond, &coder->mutex, + wait_abs) != 0; + else + mythread_cond_wait(&coder->cond, + &coder->mutex); + } + } + + return timed_out; +} + + +static lzma_ret +stream_encode_mt(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + lzma_stream_coder *coder = coder_ptr; + + switch (coder->sequence) { + case SEQ_STREAM_HEADER: + lzma_bufcpy(coder->header, &coder->header_pos, + sizeof(coder->header), + out, out_pos, out_size); + if (coder->header_pos < sizeof(coder->header)) + return LZMA_OK; + + coder->header_pos = 0; + coder->sequence = SEQ_BLOCK; + + // Fall through + + case SEQ_BLOCK: { + // Initialized to silence warnings. + lzma_vli unpadded_size = 0; + lzma_vli uncompressed_size = 0; + lzma_ret ret = LZMA_OK; + + // These are for wait_for_work(). + bool has_blocked = false; + mythread_condtime wait_abs; + + while (true) { + mythread_sync(coder->mutex) { + // Check for Block encoder errors. + ret = coder->thread_error; + if (ret != LZMA_OK) { + assert(ret != LZMA_STREAM_END); + break; + } + + // Try to read compressed data to out[]. + ret = lzma_outq_read(&coder->outq, + out, out_pos, out_size, + &unpadded_size, + &uncompressed_size); + } + + if (ret == LZMA_STREAM_END) { + // End of Block. Add it to the Index. + ret = lzma_index_append(coder->index, + allocator, unpadded_size, + uncompressed_size); + + // If we didn't fill the output buffer yet, + // try to read more data. Maybe the next + // outbuf has been finished already too. + if (*out_pos < out_size) + continue; + } + + if (ret != LZMA_OK) { + // coder->thread_error was set or + // lzma_index_append() failed. + threads_stop(coder, false); + return ret; + } + + // Try to give uncompressed data to a worker thread. + ret = stream_encode_in(coder, allocator, + in, in_pos, in_size, action); + if (ret != LZMA_OK) { + threads_stop(coder, false); + return ret; + } + + // See if we should wait or return. + // + // TODO: LZMA_SYNC_FLUSH and LZMA_SYNC_BARRIER. + if (*in_pos == in_size) { + // LZMA_RUN: More data is probably coming + // so return to let the caller fill the + // input buffer. + if (action == LZMA_RUN) + return LZMA_OK; + + // LZMA_FULL_BARRIER: The same as with + // LZMA_RUN but tell the caller that the + // barrier was completed. + if (action == LZMA_FULL_BARRIER) + return LZMA_STREAM_END; + + // Finishing or flushing isn't completed until + // all input data has been encoded and copied + // to the output buffer. + if (lzma_outq_is_empty(&coder->outq)) { + // LZMA_FINISH: Continue to encode + // the Index field. + if (action == LZMA_FINISH) + break; + + // LZMA_FULL_FLUSH: Return to tell + // the caller that flushing was + // completed. + if (action == LZMA_FULL_FLUSH) + return LZMA_STREAM_END; + } + } + + // Return if there is no output space left. + // This check must be done after testing the input + // buffer, because we might want to use a different + // return code. + if (*out_pos == out_size) + return LZMA_OK; + + // Neither in nor out has been used completely. + // Wait until there's something we can do. + if (wait_for_work(coder, &wait_abs, &has_blocked, + *in_pos < in_size)) + return LZMA_TIMED_OUT; + } + + // All Blocks have been encoded and the threads have stopped. + // Prepare to encode the Index field. + return_if_error(lzma_index_encoder_init( + &coder->index_encoder, allocator, + coder->index)); + coder->sequence = SEQ_INDEX; + + // Update the progress info to take the Index and + // Stream Footer into account. Those are very fast to encode + // so in terms of progress information they can be thought + // to be ready to be copied out. + coder->progress_out += lzma_index_size(coder->index) + + LZMA_STREAM_HEADER_SIZE; + } + + // Fall through + + case SEQ_INDEX: { + // Call the Index encoder. It doesn't take any input, so + // those pointers can be NULL. + const lzma_ret ret = coder->index_encoder.code( + coder->index_encoder.coder, allocator, + NULL, NULL, 0, + out, out_pos, out_size, LZMA_RUN); + if (ret != LZMA_STREAM_END) + return ret; + + // Encode the Stream Footer into coder->buffer. + coder->stream_flags.backward_size + = lzma_index_size(coder->index); + if (lzma_stream_footer_encode(&coder->stream_flags, + coder->header) != LZMA_OK) + return LZMA_PROG_ERROR; + + coder->sequence = SEQ_STREAM_FOOTER; + } + + // Fall through + + case SEQ_STREAM_FOOTER: + lzma_bufcpy(coder->header, &coder->header_pos, + sizeof(coder->header), + out, out_pos, out_size); + return coder->header_pos < sizeof(coder->header) + ? LZMA_OK : LZMA_STREAM_END; + } + + assert(0); + return LZMA_PROG_ERROR; +} + + +static void +stream_encoder_mt_end(void *coder_ptr, const lzma_allocator *allocator) +{ + lzma_stream_coder *coder = coder_ptr; + + // Threads must be killed before the output queue can be freed. + threads_end(coder, allocator); + lzma_outq_end(&coder->outq, allocator); + + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + lzma_free(coder->filters[i].options, allocator); + + lzma_next_end(&coder->index_encoder, allocator); + lzma_index_end(coder->index, allocator); + + mythread_cond_destroy(&coder->cond); + mythread_mutex_destroy(&coder->mutex); + + lzma_free(coder, allocator); + return; +} + + +/// Options handling for lzma_stream_encoder_mt_init() and +/// lzma_stream_encoder_mt_memusage() +static lzma_ret +get_options(const lzma_mt *options, lzma_options_easy *opt_easy, + const lzma_filter **filters, uint64_t *block_size, + uint64_t *outbuf_size_max) +{ + // Validate some of the options. + if (options == NULL) + return LZMA_PROG_ERROR; + + if (options->flags != 0 || options->threads == 0 + || options->threads > LZMA_THREADS_MAX) + return LZMA_OPTIONS_ERROR; + + if (options->filters != NULL) { + // Filter chain was given, use it as is. + *filters = options->filters; + } else { + // Use a preset. + if (lzma_easy_preset(opt_easy, options->preset)) + return LZMA_OPTIONS_ERROR; + + *filters = opt_easy->filters; + } + + // Block size + if (options->block_size > 0) { + if (options->block_size > BLOCK_SIZE_MAX) + return LZMA_OPTIONS_ERROR; + + *block_size = options->block_size; + } else { + // Determine the Block size from the filter chain. + *block_size = lzma_mt_block_size(*filters); + if (*block_size == 0) + return LZMA_OPTIONS_ERROR; + + assert(*block_size <= BLOCK_SIZE_MAX); + } + + // Calculate the maximum amount output that a single output buffer + // may need to hold. This is the same as the maximum total size of + // a Block. + *outbuf_size_max = lzma_block_buffer_bound64(*block_size); + if (*outbuf_size_max == 0) + return LZMA_MEM_ERROR; + + return LZMA_OK; +} + + +static void +get_progress(void *coder_ptr, uint64_t *progress_in, uint64_t *progress_out) +{ + lzma_stream_coder *coder = coder_ptr; + + // Lock coder->mutex to prevent finishing threads from moving their + // progress info from the worker_thread structure to lzma_stream_coder. + mythread_sync(coder->mutex) { + *progress_in = coder->progress_in; + *progress_out = coder->progress_out; + + for (size_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + *progress_in += coder->threads[i].progress_in; + *progress_out += coder->threads[i] + .progress_out; + } + } + } + + return; +} + + +static lzma_ret +stream_encoder_mt_init(lzma_next_coder *next, const lzma_allocator *allocator, + const lzma_mt *options) +{ + lzma_next_coder_init(&stream_encoder_mt_init, next, allocator); + + // Get the filter chain. + lzma_options_easy easy; + const lzma_filter *filters; + uint64_t block_size; + uint64_t outbuf_size_max; + return_if_error(get_options(options, &easy, &filters, + &block_size, &outbuf_size_max)); + +#if SIZE_MAX < UINT64_MAX + if (block_size > SIZE_MAX) + return LZMA_MEM_ERROR; +#endif + + // Validate the filter chain so that we can give an error in this + // function instead of delaying it to the first call to lzma_code(). + // The memory usage calculation verifies the filter chain as + // a side effect so we take advatange of that. + if (lzma_raw_encoder_memusage(filters) == UINT64_MAX) + return LZMA_OPTIONS_ERROR; + + // Validate the Check ID. + if ((unsigned int)(options->check) > LZMA_CHECK_ID_MAX) + return LZMA_PROG_ERROR; + + if (!lzma_check_is_supported(options->check)) + return LZMA_UNSUPPORTED_CHECK; + + // Allocate and initialize the base structure if needed. + lzma_stream_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) + return LZMA_MEM_ERROR; + + next->coder = coder; + + // For the mutex and condition variable initializations + // the error handling has to be done here because + // stream_encoder_mt_end() doesn't know if they have + // already been initialized or not. + if (mythread_mutex_init(&coder->mutex)) { + lzma_free(coder, allocator); + next->coder = NULL; + return LZMA_MEM_ERROR; + } + + if (mythread_cond_init(&coder->cond)) { + mythread_mutex_destroy(&coder->mutex); + lzma_free(coder, allocator); + next->coder = NULL; + return LZMA_MEM_ERROR; + } + + next->code = &stream_encode_mt; + next->end = &stream_encoder_mt_end; + next->get_progress = &get_progress; +// next->update = &stream_encoder_mt_update; + + coder->filters[0].id = LZMA_VLI_UNKNOWN; + coder->index_encoder = LZMA_NEXT_CODER_INIT; + coder->index = NULL; + memzero(&coder->outq, sizeof(coder->outq)); + coder->threads = NULL; + coder->threads_max = 0; + coder->threads_initialized = 0; + } + + // Basic initializations + coder->sequence = SEQ_STREAM_HEADER; + coder->block_size = (size_t)(block_size); + coder->thread_error = LZMA_OK; + coder->thr = NULL; + + // Allocate the thread-specific base structures. + assert(options->threads > 0); + if (coder->threads_max != options->threads) { + threads_end(coder, allocator); + + coder->threads = NULL; + coder->threads_max = 0; + + coder->threads_initialized = 0; + coder->threads_free = NULL; + + coder->threads = lzma_alloc( + options->threads * sizeof(worker_thread), + allocator); + if (coder->threads == NULL) + return LZMA_MEM_ERROR; + + coder->threads_max = options->threads; + } else { + // Reuse the old structures and threads. Tell the running + // threads to stop and wait until they have stopped. + threads_stop(coder, true); + } + + // Output queue + return_if_error(lzma_outq_init(&coder->outq, allocator, + outbuf_size_max, options->threads)); + + // Timeout + coder->timeout = options->timeout; + + // Free the old filter chain and copy the new one. + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + lzma_free(coder->filters[i].options, allocator); + + return_if_error(lzma_filters_copy( + filters, coder->filters, allocator)); + + // Index + lzma_index_end(coder->index, allocator); + coder->index = lzma_index_init(allocator); + if (coder->index == NULL) + return LZMA_MEM_ERROR; + + // Stream Header + coder->stream_flags.version = 0; + coder->stream_flags.check = options->check; + return_if_error(lzma_stream_header_encode( + &coder->stream_flags, coder->header)); + + coder->header_pos = 0; + + // Progress info + coder->progress_in = 0; + coder->progress_out = LZMA_STREAM_HEADER_SIZE; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_stream_encoder_mt(lzma_stream *strm, const lzma_mt *options) +{ + lzma_next_strm_init(stream_encoder_mt_init, strm, options); + + strm->internal->supported_actions[LZMA_RUN] = true; +// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_BARRIER] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} + + +// This function name is a monster but it's consistent with the older +// monster names. :-( 31 chars is the max that C99 requires so in that +// sense it's not too long. ;-) +extern LZMA_API(uint64_t) +lzma_stream_encoder_mt_memusage(const lzma_mt *options) +{ + lzma_options_easy easy; + const lzma_filter *filters; + uint64_t block_size; + uint64_t outbuf_size_max; + + if (get_options(options, &easy, &filters, &block_size, + &outbuf_size_max) != LZMA_OK) + return UINT64_MAX; + + // Memory usage of the input buffers + const uint64_t inbuf_memusage = options->threads * block_size; + + // Memory usage of the filter encoders + uint64_t filters_memusage = lzma_raw_encoder_memusage(filters); + if (filters_memusage == UINT64_MAX) + return UINT64_MAX; + + filters_memusage *= options->threads; + + // Memory usage of the output queue + const uint64_t outq_memusage = lzma_outq_memusage( + outbuf_size_max, options->threads); + if (outq_memusage == UINT64_MAX) + return UINT64_MAX; + + // Sum them with overflow checking. + uint64_t total_memusage = LZMA_MEMUSAGE_BASE + + sizeof(lzma_stream_coder) + + options->threads * sizeof(worker_thread); + + if (UINT64_MAX - total_memusage < inbuf_memusage) + return UINT64_MAX; + + total_memusage += inbuf_memusage; + + if (UINT64_MAX - total_memusage < filters_memusage) + return UINT64_MAX; + + total_memusage += filters_memusage; + + if (UINT64_MAX - total_memusage < outq_memusage) + return UINT64_MAX; + + return total_memusage + outq_memusage; +} diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c index 8cf48a4..1bc2f97 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c @@ -30,15 +30,13 @@ stream_flags_decode(lzma_stream_flags *options, const uint8_t *in) extern LZMA_API(lzma_ret) lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in) { - uint32_t crc; - // Magic if (memcmp(in, lzma_header_magic, sizeof(lzma_header_magic)) != 0) return LZMA_FORMAT_ERROR; // Verify the CRC32 so we can distinguish between corrupt // and unsupported files. - crc = lzma_crc32(in + sizeof(lzma_header_magic), + const uint32_t crc = lzma_crc32(in + sizeof(lzma_header_magic), LZMA_STREAM_FLAGS_SIZE, 0); if (crc != unaligned_read32le(in + sizeof(lzma_header_magic) + LZMA_STREAM_FLAGS_SIZE)) @@ -61,15 +59,13 @@ lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in) extern LZMA_API(lzma_ret) lzma_stream_footer_decode(lzma_stream_flags *options, const uint8_t *in) { - uint32_t crc; - // Magic if (memcmp(in + sizeof(uint32_t) * 2 + LZMA_STREAM_FLAGS_SIZE, lzma_footer_magic, sizeof(lzma_footer_magic)) != 0) return LZMA_FORMAT_ERROR; // CRC32 - crc = lzma_crc32(in + sizeof(uint32_t), + const uint32_t crc = lzma_crc32(in + sizeof(uint32_t), sizeof(uint32_t) + LZMA_STREAM_FLAGS_SIZE, 0); if (crc != unaligned_read32le(in)) return LZMA_DATA_ERROR; diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c index 290339e..4e71715 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c @@ -29,8 +29,6 @@ stream_flags_encode(const lzma_stream_flags *options, uint8_t *out) extern LZMA_API(lzma_ret) lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) { - uint32_t crc; - assert(sizeof(lzma_header_magic) + LZMA_STREAM_FLAGS_SIZE + 4 == LZMA_STREAM_HEADER_SIZE); @@ -45,7 +43,7 @@ lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) return LZMA_PROG_ERROR; // CRC32 of the Stream Header - crc = lzma_crc32(out + sizeof(lzma_header_magic), + const uint32_t crc = lzma_crc32(out + sizeof(lzma_header_magic), LZMA_STREAM_FLAGS_SIZE, 0); unaligned_write32le(out + sizeof(lzma_header_magic) @@ -58,8 +56,6 @@ lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) extern LZMA_API(lzma_ret) lzma_stream_footer_encode(const lzma_stream_flags *options, uint8_t *out) { - uint32_t crc; - assert(2 * 4 + LZMA_STREAM_FLAGS_SIZE + sizeof(lzma_footer_magic) == LZMA_STREAM_HEADER_SIZE); @@ -77,7 +73,7 @@ lzma_stream_footer_encode(const lzma_stream_flags *options, uint8_t *out) return LZMA_PROG_ERROR; // CRC32 - crc = lzma_crc32( + const uint32_t crc = lzma_crc32( out + 4, 4 + LZMA_STREAM_FLAGS_SIZE, 0); unaligned_write32le(out, crc); diff --git a/Utilities/cmliblzma/liblzma/common/vli_decoder.c b/Utilities/cmliblzma/liblzma/common/vli_decoder.c index 1c66384..c181828 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/vli_decoder.c @@ -14,8 +14,8 @@ extern LZMA_API(lzma_ret) -lzma_vli_decode(lzma_vli *LZMA_RESTRICT vli, size_t *vli_pos, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lzma_vli_decode(lzma_vli *restrict vli, size_t *vli_pos, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size) { // If we haven't been given vli_pos, work in single-call mode. diff --git a/Utilities/cmliblzma/liblzma/common/vli_encoder.c b/Utilities/cmliblzma/liblzma/common/vli_encoder.c index 09e90cb..f864269 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/vli_encoder.c @@ -15,7 +15,7 @@ extern LZMA_API(lzma_ret) lzma_vli_encode(lzma_vli vli, size_t *vli_pos, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { // If we haven't been given vli_pos, work in single-call mode. diff --git a/Utilities/cmliblzma/liblzma/common/vli_size.c b/Utilities/cmliblzma/liblzma/common/vli_size.c index 8b931e4..ec1b4fa 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_size.c +++ b/Utilities/cmliblzma/liblzma/common/vli_size.c @@ -16,11 +16,10 @@ extern LZMA_API(uint32_t) lzma_vli_size(lzma_vli vli) { - uint32_t i = 0; - if (vli > LZMA_VLI_MAX) return 0; + uint32_t i = 0; do { vli >>= 7; ++i; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_common.c b/Utilities/cmliblzma/liblzma/delta/delta_common.c index 803e674..4768201 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_common.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_common.c @@ -15,8 +15,9 @@ static void -delta_coder_end(lzma_coder *coder, lzma_allocator *allocator) +delta_coder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_delta_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -24,20 +25,21 @@ delta_coder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_delta_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { - const lzma_options_delta *opt; - // Allocate memory for the decoder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_delta_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_delta_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; + // End function is the same for encoder and decoder. next->end = &delta_coder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Validate the options. @@ -45,16 +47,15 @@ lzma_delta_coder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_OPTIONS_ERROR; // Set the delta distance. - opt = filters[0].options; - next->coder->distance = opt->dist; + const lzma_options_delta *opt = filters[0].options; + coder->distance = opt->dist; // Initialize the rest of the variables. - next->coder->pos = 0; - memzero(next->coder->history, LZMA_DELTA_DIST_MAX); + coder->pos = 0; + memzero(coder->history, LZMA_DELTA_DIST_MAX); // Initialize the next decoder in the chain, if any. - return lzma_next_filter_init(&next->coder->next, - allocator, filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } @@ -68,5 +69,5 @@ lzma_delta_coder_memusage(const void *options) || opt->dist > LZMA_DELTA_DIST_MAX) return UINT64_MAX; - return sizeof(lzma_coder); + return sizeof(lzma_delta_coder); } diff --git a/Utilities/cmliblzma/liblzma/delta/delta_decoder.c b/Utilities/cmliblzma/liblzma/delta/delta_decoder.c index 28df727..6859afa 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_decoder.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_decoder.c @@ -15,12 +15,11 @@ static void -decode_buffer(lzma_coder *coder, uint8_t *buffer, size_t size) +decode_buffer(lzma_delta_coder *coder, uint8_t *buffer, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { buffer[i] += coder->history[(distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = buffer[i]; } @@ -28,17 +27,18 @@ decode_buffer(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -delta_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +delta_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { - const size_t out_start = *out_pos; - lzma_ret ret; + lzma_delta_coder *coder = coder_ptr; assert(coder->next.code != NULL); - ret = coder->next.code(coder->next.coder, allocator, + const size_t out_start = *out_pos; + + const lzma_ret ret = coder->next.code(coder->next.coder, allocator, in, in_pos, in_size, out, out_pos, out_size, action); @@ -49,7 +49,7 @@ delta_decode(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { next->code = &delta_decode; @@ -58,15 +58,14 @@ lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_delta_props_decode(void **options, lzma_allocator *allocator, +lzma_delta_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_delta *opt; - if (props_size != 1) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_delta), allocator); + lzma_options_delta *opt + = lzma_alloc(sizeof(lzma_options_delta), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_decoder.h b/Utilities/cmliblzma/liblzma/delta/delta_decoder.h index ae89acc..ad89cc6 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_decoder.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_decoder.h @@ -16,10 +16,11 @@ #include "delta_common.h" extern lzma_ret lzma_delta_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_delta_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/delta/delta_encoder.c b/Utilities/cmliblzma/liblzma/delta/delta_encoder.c index a39c154..3841651 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_encoder.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_encoder.c @@ -18,13 +18,12 @@ /// is the first filter in the chain (and thus the last filter in the /// encoder's filter stack). static void -copy_and_encode(lzma_coder *coder, - const uint8_t *LZMA_RESTRICT in, uint8_t *LZMA_RESTRICT out, size_t size) +copy_and_encode(lzma_delta_coder *coder, + const uint8_t *restrict in, uint8_t *restrict out, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { const uint8_t tmp = coder->history[ (distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = in[i]; @@ -36,12 +35,11 @@ copy_and_encode(lzma_coder *coder, /// Encodes the data in place. This is used when we are the last filter /// in the chain (and thus non-last filter in the encoder's filter stack). static void -encode_in_place(lzma_coder *coder, uint8_t *buffer, size_t size) +encode_in_place(lzma_delta_coder *coder, uint8_t *buffer, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { const uint8_t tmp = coder->history[ (distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = buffer[i]; @@ -51,11 +49,13 @@ encode_in_place(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -delta_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +delta_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_delta_coder *coder = coder_ptr; + lzma_ret ret; if (coder->next.code == NULL) { @@ -86,10 +86,12 @@ delta_encode(lzma_coder *coder, lzma_allocator *allocator, static lzma_ret -delta_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +delta_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_delta_coder *coder = coder_ptr; + // Delta doesn't and will never support changing the options in // the middle of encoding. If the app tries to change them, we // simply ignore them. @@ -99,7 +101,7 @@ delta_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { next->code = &delta_encode; @@ -111,13 +113,12 @@ lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out) { - const lzma_options_delta *opt = options; - // The caller must have already validated the options, so it's // LZMA_PROG_ERROR if they are invalid. if (lzma_delta_coder_memusage(options) == UINT64_MAX) return LZMA_PROG_ERROR; + const lzma_options_delta *opt = options; out[0] = opt->dist - LZMA_DELTA_DIST_MIN; return LZMA_OK; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_encoder.h b/Utilities/cmliblzma/liblzma/delta/delta_encoder.h index a447862..4ab9847 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_encoder.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_encoder.h @@ -16,7 +16,8 @@ #include "delta_common.h" extern lzma_ret lzma_delta_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out); diff --git a/Utilities/cmliblzma/liblzma/delta/delta_private.h b/Utilities/cmliblzma/liblzma/delta/delta_private.h index 62b7fed..0d6cb38 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_private.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_private.h @@ -15,7 +15,7 @@ #include "delta_common.h" -struct lzma_coder_s { +typedef struct { /// Next coder in the chain lzma_next_coder next; @@ -27,11 +27,11 @@ struct lzma_coder_s { /// Buffer to hold history of the original data uint8_t history[LZMA_DELTA_DIST_MAX]; -}; +} lzma_delta_coder; extern lzma_ret lzma_delta_coder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/liblzma.pc.in b/Utilities/cmliblzma/liblzma/liblzma.pc.in index 7f11f1a..9fa4891 100644 --- a/Utilities/cmliblzma/liblzma/liblzma.pc.in +++ b/Utilities/cmliblzma/liblzma/liblzma.pc.in @@ -16,4 +16,4 @@ URL: @PACKAGE_URL@ Version: @PACKAGE_VERSION@ Cflags: -I${includedir} Libs: -L${libdir} -llzma -Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ +Libs.private: @PTHREAD_CFLAGS@ @LIBS@ diff --git a/Utilities/cmliblzma/liblzma/lz/lz_decoder.c b/Utilities/cmliblzma/liblzma/lz/lz_decoder.c index 9fa1bdc..c708644 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_decoder.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_decoder.c @@ -20,7 +20,7 @@ #include "lz_decoder.h" -struct lzma_coder_s { +typedef struct { /// Dictionary (history buffer) lzma_dict dict; @@ -48,7 +48,7 @@ struct lzma_coder_s { size_t size; uint8_t buffer[LZMA_BUFFER_SIZE]; } temp; -}; +} lzma_coder; static void @@ -64,22 +64,18 @@ lz_decoder_reset(lzma_coder *coder) static lzma_ret decode_buffer(lzma_coder *coder, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size) + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) { while (true) { - size_t copy_size; - size_t dict_start; - lzma_ret ret; - // Wrap the dictionary if needed. if (coder->dict.pos == coder->dict.size) coder->dict.pos = 0; // Store the current dictionary position. It is needed to know // where to start copying to the out[] buffer. - dict_start = coder->dict.pos; + const size_t dict_start = coder->dict.pos; // Calculate how much we allow coder->lz.code() to decode. // It must not decode past the end of the dictionary @@ -90,13 +86,13 @@ decode_buffer(lzma_coder *coder, coder->dict.size - coder->dict.pos); // Call the coder->lz.code() to do the actual decoding. - ret = coder->lz.code( + const lzma_ret ret = coder->lz.code( coder->lz.coder, &coder->dict, in, in_pos, in_size); // Copy the decoded data from the dictionary to the out[] // buffer. - copy_size = coder->dict.pos - dict_start; + const size_t copy_size = coder->dict.pos - dict_start; assert(copy_size <= out_size - *out_pos); memcpy(out + *out_pos, coder->dict.buf + dict_start, copy_size); @@ -129,13 +125,15 @@ decode_buffer(lzma_coder *coder, static lzma_ret -lz_decode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +lz_decode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_coder *coder = coder_ptr; + if (coder->next.code == NULL) return decode_buffer(coder, in, in_pos, in_size, out, out_pos, out_size); @@ -143,15 +141,13 @@ lz_decode(lzma_coder *coder, // We aren't the last coder in the chain, we need to decode // our input to a temporary buffer. while (*out_pos < out_size) { - lzma_ret ret; - // Fill the temporary buffer if it is empty. if (!coder->next_finished && coder->temp.pos == coder->temp.size) { coder->temp.pos = 0; coder->temp.size = 0; - ret = coder->next.code( + const lzma_ret ret = coder->next.code( coder->next.coder, allocator, in, in_pos, in_size, coder->temp.buffer, &coder->temp.size, @@ -173,7 +169,7 @@ lz_decode(lzma_coder *coder, return LZMA_OK; } - ret = decode_buffer(coder, coder->temp.buffer, + const lzma_ret ret = decode_buffer(coder, coder->temp.buffer, &coder->temp.pos, coder->temp.size, out, out_pos, out_size); @@ -190,8 +186,10 @@ lz_decode(lzma_coder *coder, static void -lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +lz_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_coder *coder = coder_ptr; + lzma_next_end(&coder->next, allocator); lzma_free(coder->dict.buf, allocator); @@ -206,32 +204,33 @@ lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_decoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)) { - lzma_lz_options lz_options; - // Allocate the base structure if it isn't already allocated. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &lz_decode; next->end = &lz_decoder_end; - next->coder->dict.buf = NULL; - next->coder->dict.size = 0; - next->coder->lz = LZMA_LZ_DECODER_INIT; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->dict.buf = NULL; + coder->dict.size = 0; + coder->lz = LZMA_LZ_DECODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Allocate and initialize the LZ-based decoder. It will also give // us the dictionary size. - return_if_error(lz_init(&next->coder->lz, allocator, + lzma_lz_options lz_options; + return_if_error(lz_init(&coder->lz, allocator, filters[0].options, &lz_options)); // If the dictionary size is very small, increase it to 4096 bytes. @@ -255,14 +254,14 @@ lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, lz_options.dict_size = (lz_options.dict_size + 15) & ~((size_t)(15)); // Allocate and initialize the dictionary. - if (next->coder->dict.size != lz_options.dict_size) { - lzma_free(next->coder->dict.buf, allocator); - next->coder->dict.buf + if (coder->dict.size != lz_options.dict_size) { + lzma_free(coder->dict.buf, allocator); + coder->dict.buf = lzma_alloc(lz_options.dict_size, allocator); - if (next->coder->dict.buf == NULL) + if (coder->dict.buf == NULL) return LZMA_MEM_ERROR; - next->coder->dict.size = lz_options.dict_size; + coder->dict.size = lz_options.dict_size; } lz_decoder_reset(next->coder); @@ -275,21 +274,20 @@ lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, const size_t copy_size = my_min(lz_options.preset_dict_size, lz_options.dict_size); const size_t offset = lz_options.preset_dict_size - copy_size; - memcpy(next->coder->dict.buf, lz_options.preset_dict + offset, + memcpy(coder->dict.buf, lz_options.preset_dict + offset, copy_size); - next->coder->dict.pos = copy_size; - next->coder->dict.full = copy_size; + coder->dict.pos = copy_size; + coder->dict.full = copy_size; } // Miscellaneous initializations - next->coder->next_finished = false; - next->coder->this_finished = false; - next->coder->temp.pos = 0; - next->coder->temp.size = 0; + coder->next_finished = false; + coder->this_finished = false; + coder->temp.pos = 0; + coder->temp.size = 0; // Initialize the next filter in the chain, if any. - return lzma_next_filter_init(&next->coder->next, allocator, - filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } @@ -301,7 +299,8 @@ lzma_lz_decoder_memusage(size_t dictionary_size) extern void -lzma_lz_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) +lzma_lz_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) { + lzma_coder *coder = coder_ptr; coder->lz.set_uncompressed(coder->lz.coder, uncompressed_size); } diff --git a/Utilities/cmliblzma/liblzma/lz/lz_decoder.h b/Utilities/cmliblzma/liblzma/lz/lz_decoder.h index 76011f2..754ccf3 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_decoder.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_decoder.h @@ -53,45 +53,45 @@ typedef struct { typedef struct { /// Data specific to the LZ-based decoder - lzma_coder *coder; + void *coder; /// Function to decode from in[] to *dict - lzma_ret (*code)(lzma_coder *LZMA_RESTRICT coder, - lzma_dict *LZMA_RESTRICT dict, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size); + lzma_ret (*code)(void *coder, + lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size); - void (*reset)(lzma_coder *coder, const void *options); + void (*reset)(void *coder, const void *options); /// Set the uncompressed size - void (*set_uncompressed)(lzma_coder *coder, - lzma_vli uncompressed_size); + void (*set_uncompressed)(void *coder, lzma_vli uncompressed_size); /// Free allocated resources - void (*end)(lzma_coder *coder, lzma_allocator *allocator); + void (*end)(void *coder, const lzma_allocator *allocator); } lzma_lz_decoder; -static const lzma_lz_decoder LZMA_LZ_DECODER_INIT = - { - NULL, - NULL, - NULL, - NULL, - NULL, - }; +#define LZMA_LZ_DECODER_INIT \ + (lzma_lz_decoder){ \ + .coder = NULL, \ + .code = NULL, \ + .reset = NULL, \ + .set_uncompressed = NULL, \ + .end = NULL, \ + } extern lzma_ret lzma_lz_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters, + const lzma_allocator *allocator, + const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_decoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)); extern uint64_t lzma_lz_decoder_memusage(size_t dictionary_size); extern void lzma_lz_decoder_uncompressed( - lzma_coder *coder, lzma_vli uncompressed_size); + void *coder, lzma_vli uncompressed_size); ////////////////////// @@ -151,15 +151,13 @@ dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len) dict->pos += left; } else { - uint32_t copy_pos; - uint32_t copy_size; - // The bigger the dictionary, the more rare this // case occurs. We need to "wrap" the dict, thus // we might need two memcpy() to copy all the data. assert(dict->full == dict->size); - copy_pos = dict->pos - distance - 1 + dict->size; - copy_size = dict->size - copy_pos; + const uint32_t copy_pos + = dict->pos - distance - 1 + dict->size; + uint32_t copy_size = dict->size - copy_pos; if (copy_size < left) { memmove(dict->buf + dict->pos, dict->buf + copy_pos, @@ -202,9 +200,9 @@ dict_put(lzma_dict *dict, uint8_t byte) /// Copies arbitrary amount of data into the dictionary. static inline void -dict_write(lzma_dict *LZMA_RESTRICT dict, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size, - size_t *LZMA_RESTRICT left) +dict_write(lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size, + size_t *restrict left) { // NOTE: If we are being given more data than the size of the // dictionary, it could be possible to optimize the LZ decoder diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder.c b/Utilities/cmliblzma/liblzma/lz/lz_encoder.c index 1dae924..9a74b7c 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder.c @@ -20,8 +20,10 @@ # include "lz_encoder_hash_table.h" #endif +#include "memcmplen.h" -struct lzma_coder_s { + +typedef struct { /// LZ-based encoder e.g. LZMA lzma_lz_encoder lz; @@ -30,7 +32,7 @@ struct lzma_coder_s { /// Next coder in the chain lzma_next_coder next; -}; +} lzma_coder; /// \brief Moves the data in the input window to free space for new data @@ -43,18 +45,16 @@ struct lzma_coder_s { static void move_window(lzma_mf *mf) { - uint32_t move_offset; - size_t move_size; - // Align the move to a multiple of 16 bytes. Some LZ-based encoders // like LZMA use the lowest bits of mf->read_pos to know the // alignment of the uncompressed data. We also get better speed // for memmove() with aligned buffers. assert(mf->read_pos > mf->keep_size_before); - move_offset = (mf->read_pos - mf->keep_size_before) & ~UINT32_C(15); + const uint32_t move_offset + = (mf->read_pos - mf->keep_size_before) & ~UINT32_C(15); assert(mf->write_pos > move_offset); - move_size = mf->write_pos - move_offset; + const size_t move_size = mf->write_pos - move_offset; assert(move_offset + move_size <= mf->size); @@ -78,12 +78,10 @@ move_window(lzma_mf *mf) /// This function must not be called once it has returned LZMA_STREAM_END. /// static lzma_ret -fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, - size_t *in_pos, size_t in_size, lzma_action action) +fill_window(lzma_coder *coder, const lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size, + lzma_action action) { - size_t write_pos; - lzma_ret ret; - assert(coder->mf.read_pos <= coder->mf.write_pos); // Move the sliding window if needed. @@ -93,7 +91,8 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, // Maybe this is ugly, but lzma_mf uses uint32_t for most things // (which I find cleanest), but we need size_t here when filling // the history window. - write_pos = coder->mf.write_pos; + size_t write_pos = coder->mf.write_pos; + lzma_ret ret; if (coder->next.code == NULL) { // Not using a filter, simply memcpy() as much as possible. lzma_bufcpy(in, in_pos, in_size, coder->mf.buffer, @@ -111,6 +110,12 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, coder->mf.write_pos = write_pos; + // Silence Valgrind. lzma_memcmplen() can read extra bytes + // and Valgrind will give warnings if those bytes are uninitialized + // because Valgrind cannot see that the values of the uninitialized + // bytes are eventually ignored. + memzero(coder->mf.buffer + write_pos, LZMA_MEMCMPLEN_EXTRA); + // If end of stream has been reached or flushing completed, we allow // the encoder to process all the input (that is, read_pos is allowed // to reach write_pos). Otherwise we keep keep_size_after bytes @@ -134,7 +139,7 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, && coder->mf.read_pos < coder->mf.read_limit) { // Match finder may update coder->pending and expects it to // start from zero, so use a temporary variable. - const size_t pending = coder->mf.pending; + const uint32_t pending = coder->mf.pending; coder->mf.pending = 0; // Rewind read_pos so that the match finder can hash @@ -152,16 +157,16 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, static lzma_ret -lz_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lz_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_coder *coder = coder_ptr; + while (*out_pos < out_size && (*in_pos < in_size || action != LZMA_RUN)) { - lzma_ret ret; - // Read more data to coder->mf.buffer if needed. if (coder->mf.action == LZMA_RUN && coder->mf.read_pos >= coder->mf.read_limit) @@ -169,7 +174,7 @@ lz_encode(lzma_coder *coder, lzma_allocator *allocator, in, in_pos, in_size, action)); // Encode - ret = coder->lz.code(coder->lz.coder, + const lzma_ret ret = coder->lz.code(coder->lz.coder, &coder->mf, out, out_pos, out_size); if (ret != LZMA_OK) { // Setting this to LZMA_RUN for cases when we are @@ -185,17 +190,9 @@ lz_encode(lzma_coder *coder, lzma_allocator *allocator, static bool -lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, +lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator, const lzma_lz_options *lz_options) { - bool is_bt; - uint32_t new_count; - uint32_t reserve; - uint32_t old_size; - uint32_t hash_bytes; - uint32_t hs; - uint32_t old_count; - // For now, the dictionary size is limited to 1.5 GiB. This may grow // in the future if needed, but it needs a little more work than just // changing this check. @@ -221,14 +218,14 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, // to size_t. // - Memory usage calculation needs something too, e.g. use uint64_t // for mf->size. - reserve = lz_options->dict_size / 2; + uint32_t reserve = lz_options->dict_size / 2; if (reserve > (UINT32_C(1) << 30)) reserve /= 2; reserve += (lz_options->before_size + lz_options->match_len_max + lz_options->after_size) / 2 + (UINT32_C(1) << 19); - old_size = mf->size; + const uint32_t old_size = mf->size; mf->size = mf->keep_size_before + reserve + mf->keep_size_after; // Deallocate the old history buffer if it exists but has different @@ -298,11 +295,12 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, // Calculate the sizes of mf->hash and mf->son and check that // nice_len is big enough for the selected match finder. - hash_bytes = lz_options->match_finder & 0x0F; + const uint32_t hash_bytes = lz_options->match_finder & 0x0F; if (hash_bytes > mf->nice_len) return true; - is_bt = (lz_options->match_finder & 0x10) != 0; + const bool is_bt = (lz_options->match_finder & 0x10) != 0; + uint32_t hs; if (hash_bytes == 2) { hs = 0xFFFF; @@ -338,25 +336,22 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, hs += HASH_4_SIZE; */ - // If the above code calculating hs is modified, make sure that - // this assertion stays valid (UINT32_MAX / 5 is not strictly the - // exact limit). If it doesn't, you need to calculate that - // hash_size_sum + sons_count cannot overflow. - assert(hs < UINT32_MAX / 5); - - old_count = mf->hash_size_sum + mf->sons_count; - mf->hash_size_sum = hs; + const uint32_t old_hash_count = mf->hash_count; + const uint32_t old_sons_count = mf->sons_count; + mf->hash_count = hs; mf->sons_count = mf->cyclic_size; if (is_bt) mf->sons_count *= 2; - new_count = mf->hash_size_sum + mf->sons_count; - // Deallocate the old hash array if it exists and has different size // than what is needed now. - if (old_count != new_count) { + if (old_hash_count != mf->hash_count + || old_sons_count != mf->sons_count) { lzma_free(mf->hash, allocator); mf->hash = NULL; + + lzma_free(mf->son, allocator); + mf->son = NULL; } // Maximum number of match finder cycles @@ -373,16 +368,23 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, static bool -lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator, +lz_encoder_init(lzma_mf *mf, const lzma_allocator *allocator, const lzma_lz_options *lz_options) { - size_t alloc_count; - // Allocate the history buffer. if (mf->buffer == NULL) { - mf->buffer = lzma_alloc(mf->size, allocator); + // lzma_memcmplen() is used for the dictionary buffer + // so we need to allocate a few extra bytes to prevent + // it from reading past the end of the buffer. + mf->buffer = lzma_alloc(mf->size + LZMA_MEMCMPLEN_EXTRA, + allocator); if (mf->buffer == NULL) return true; + + // Keep Valgrind happy with lzma_memcmplen() and initialize + // the extra bytes whose value may get read but which will + // effectively get ignored. + memzero(mf->buffer + mf->size, LZMA_MEMCMPLEN_EXTRA); } // Use cyclic_size as initial mf->offset. This allows @@ -396,43 +398,48 @@ lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator, mf->write_pos = 0; mf->pending = 0; - // Allocate match finder's hash array. - alloc_count = mf->hash_size_sum + mf->sons_count; - #if UINT32_MAX >= SIZE_MAX / 4 // Check for integer overflow. (Huge dictionaries are not // possible on 32-bit CPU.) - if (alloc_count > SIZE_MAX / sizeof(uint32_t)) + if (mf->hash_count > SIZE_MAX / sizeof(uint32_t) + || mf->sons_count > SIZE_MAX / sizeof(uint32_t)) return true; #endif + // Allocate and initialize the hash table. Since EMPTY_HASH_VALUE + // is zero, we can use lzma_alloc_zero() or memzero() for mf->hash. + // + // We don't need to initialize mf->son, but not doing that may + // make Valgrind complain in normalization (see normalize() in + // lz_encoder_mf.c). Skipping the initialization is *very* good + // when big dictionary is used but only small amount of data gets + // actually compressed: most of the mf->son won't get actually + // allocated by the kernel, so we avoid wasting RAM and improve + // initialization speed a lot. if (mf->hash == NULL) { - mf->hash = lzma_alloc(alloc_count * sizeof(uint32_t), + mf->hash = lzma_alloc_zero(mf->hash_count * sizeof(uint32_t), + allocator); + mf->son = lzma_alloc(mf->sons_count * sizeof(uint32_t), allocator); - if (mf->hash == NULL) - return true; - } - mf->son = mf->hash + mf->hash_size_sum; - mf->cyclic_pos = 0; + if (mf->hash == NULL || mf->son == NULL) { + lzma_free(mf->hash, allocator); + mf->hash = NULL; + + lzma_free(mf->son, allocator); + mf->son = NULL; - // Initialize the hash table. Since EMPTY_HASH_VALUE is zero, we - // can use memset(). + return true; + } + } else { /* - for (uint32_t i = 0; i < hash_size_sum; ++i) - mf->hash[i] = EMPTY_HASH_VALUE; + for (uint32_t i = 0; i < mf->hash_count; ++i) + mf->hash[i] = EMPTY_HASH_VALUE; */ - memzero(mf->hash, (size_t)(mf->hash_size_sum) * sizeof(uint32_t)); + memzero(mf->hash, mf->hash_count * sizeof(uint32_t)); + } - // We don't need to initialize mf->son, but not doing that will - // make Valgrind complain in normalization (see normalize() in - // lz_encoder_mf.c). - // - // Skipping this initialization is *very* good when big dictionary is - // used but only small amount of data gets actually compressed: most - // of the mf->hash won't get actually allocated by the kernel, so - // we avoid wasting RAM and improve initialization speed a lot. - //memzero(mf->son, (size_t)(mf->sons_count) * sizeof(uint32_t)); + mf->cyclic_pos = 0; // Handle preset dictionary. if (lz_options->preset_dict != NULL @@ -457,24 +464,32 @@ extern uint64_t lzma_lz_encoder_memusage(const lzma_lz_options *lz_options) { // Old buffers must not exist when calling lz_encoder_prepare(). - lzma_mf mf = { NULL }; + lzma_mf mf = { + .buffer = NULL, + .hash = NULL, + .son = NULL, + .hash_count = 0, + .sons_count = 0, + }; // Setup the size information into mf. if (lz_encoder_prepare(&mf, NULL, lz_options)) return UINT64_MAX; // Calculate the memory usage. - return (uint64_t)(mf.hash_size_sum + mf.sons_count) - * sizeof(uint32_t) - + (uint64_t)(mf.size) + sizeof(lzma_coder); + return ((uint64_t)(mf.hash_count) + mf.sons_count) * sizeof(uint32_t) + + mf.size + sizeof(lzma_coder); } static void -lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +lz_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_coder *coder = coder_ptr; + lzma_next_end(&coder->next, allocator); + lzma_free(coder->mf.son, allocator); lzma_free(coder->mf.hash, allocator); lzma_free(coder->mf.buffer, allocator); @@ -489,10 +504,12 @@ lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lz_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +lz_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_coder *coder = coder_ptr; + if (coder->lz.options_update == NULL) return LZMA_PROG_ERROR; @@ -505,58 +522,63 @@ lz_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_lz_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lz_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_encoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)) { - lzma_lz_options lz_options; - #ifdef HAVE_SMALL // We need that the CRC32 table has been initialized. lzma_crc32_init(); #endif // Allocate and initialize the base data structure. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &lz_encode; next->end = &lz_encoder_end; next->update = &lz_encoder_update; - next->coder->lz.coder = NULL; - next->coder->lz.code = NULL; - next->coder->lz.end = NULL; - - next->coder->mf.buffer = NULL; - next->coder->mf.hash = NULL; - next->coder->mf.hash_size_sum = 0; - next->coder->mf.sons_count = 0; - - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->lz.coder = NULL; + coder->lz.code = NULL; + coder->lz.end = NULL; + + // mf.size is initialized to silence Valgrind + // when used on optimized binaries (GCC may reorder + // code in a way that Valgrind gets unhappy). + coder->mf.buffer = NULL; + coder->mf.size = 0; + coder->mf.hash = NULL; + coder->mf.son = NULL; + coder->mf.hash_count = 0; + coder->mf.sons_count = 0; + + coder->next = LZMA_NEXT_CODER_INIT; } // Initialize the LZ-based encoder. - return_if_error(lz_init(&next->coder->lz, allocator, + lzma_lz_options lz_options; + return_if_error(lz_init(&coder->lz, allocator, filters[0].options, &lz_options)); - // Setup the size information into next->coder->mf and deallocate + // Setup the size information into coder->mf and deallocate // old buffers if they have wrong size. - if (lz_encoder_prepare(&next->coder->mf, allocator, &lz_options)) + if (lz_encoder_prepare(&coder->mf, allocator, &lz_options)) return LZMA_OPTIONS_ERROR; // Allocate new buffers if needed, and do the rest of // the initialization. - if (lz_encoder_init(&next->coder->mf, allocator, &lz_options)) + if (lz_encoder_init(&coder->mf, allocator, &lz_options)) return LZMA_MEM_ERROR; // Initialize the next filter in the chain, if any. - return lzma_next_filter_init(&next->coder->next, allocator, - filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder.h b/Utilities/cmliblzma/liblzma/lz/lz_encoder.h index dcb4b2c..426dcd8 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder.h @@ -119,7 +119,7 @@ struct lzma_mf_s { lzma_action action; /// Number of elements in hash[] - uint32_t hash_size_sum; + uint32_t hash_count; /// Number of elements in son[] uint32_t sons_count; @@ -191,19 +191,18 @@ typedef struct { typedef struct { /// Data specific to the LZ-based encoder - lzma_coder *coder; + void *coder; /// Function to encode from *dict to out[] - lzma_ret (*code)(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size); + lzma_ret (*code)(void *coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); /// Free allocated resources - void (*end)(lzma_coder *coder, lzma_allocator *allocator); + void (*end)(void *coder, const lzma_allocator *allocator); /// Update the options in the middle of the encoding. - lzma_ret (*options_update)(lzma_coder *coder, - const lzma_filter *filter); + lzma_ret (*options_update)(void *coder, const lzma_filter *filter); } lzma_lz_encoder; @@ -218,7 +217,7 @@ typedef struct { /// Get pointer to the first byte not ran through the match finder -static inline uint8_t * +static inline const uint8_t * mf_ptr(const lzma_mf *mf) { return mf->buffer + mf->read_pos; @@ -296,10 +295,10 @@ mf_read(lzma_mf *mf, uint8_t *out, size_t *out_pos, size_t out_size, extern lzma_ret lzma_lz_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_encoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)); diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h index de17c54..342a333 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h @@ -39,22 +39,25 @@ // Endianness doesn't matter in hash_2_calc() (no effect on the output). #ifdef TUKLIB_FAST_UNALIGNED_ACCESS # define hash_2_calc() \ - hash_value = *(const uint16_t *)(cur) + const uint32_t hash_value = *(const uint16_t *)(cur) #else # define hash_2_calc() \ - hash_value = (uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8) + const uint32_t hash_value \ + = (uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8) #endif #define hash_3_calc() \ - temp = hash_table[cur[0]] ^ cur[1]; \ - hash_2_value = temp & HASH_2_MASK; \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8)) & mf->hash_mask + const uint32_t temp = hash_table[cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & mf->hash_mask #define hash_4_calc() \ - temp = hash_table[cur[0]] ^ cur[1]; \ - hash_2_value = temp & HASH_2_MASK; \ - hash_3_value = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ + const uint32_t temp = hash_table[cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ + const uint32_t hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ ^ (hash_table[cur[3]] << 5)) & mf->hash_mask diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c b/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c index 50c3459..7852077 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c @@ -13,6 +13,7 @@ #include "lz_encoder.h" #include "lz_encoder_hash.h" +#include "memcmplen.h" /// \brief Find matches starting from the current byte @@ -32,9 +33,8 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) if (count > 0) { #ifndef NDEBUG - uint32_t i; // Validate the matches. - for (i = 0; i < count; ++i) { + for (uint32_t i = 0; i < count; ++i) { assert(matches[i].len <= mf->nice_len); assert(matches[i].dist < mf->read_pos); assert(memcmp(mf_ptr(mf) - 1, @@ -50,9 +50,6 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) // If a match of maximum search length was found, try to // extend the match to maximum possible length. if (len_best == mf->nice_len) { - uint8_t *p1; - uint8_t *p2; - // The limit for the match length is either the // maximum match length supported by the LZ-based // encoder or the number of bytes left in the @@ -63,15 +60,13 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) // Pointer to the byte we just ran through // the match finder. - p1 = mf_ptr(mf) - 1; + const uint8_t *p1 = mf_ptr(mf) - 1; // Pointer to the beginning of the match. We need -1 // here because the match distances are zero based. - p2 = p1 - matches[count - 1].dist - 1; + const uint8_t *p2 = p1 - matches[count - 1].dist - 1; - while (len_best < limit - && p1[len_best] == p2[len_best]) - ++len_best; + len_best = lzma_memcmplen(p1, p2, len_best, limit); } } @@ -112,36 +107,35 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) static void normalize(lzma_mf *mf) { - uint32_t i; - uint32_t subvalue; - uint32_t count; - uint32_t *hash; - assert(mf->read_pos + mf->offset == MUST_NORMALIZE_POS); // In future we may not want to touch the lowest bits, because there // may be match finders that use larger resolution than one byte. - subvalue = (MUST_NORMALIZE_POS - mf->cyclic_size); + const uint32_t subvalue + = (MUST_NORMALIZE_POS - mf->cyclic_size); // & (~(UINT32_C(1) << 10) - 1); - count = mf->hash_size_sum + mf->sons_count; - hash = mf->hash; - - for (i = 0; i < count; ++i) { + for (uint32_t i = 0; i < mf->hash_count; ++i) { // If the distance is greater than the dictionary size, // we can simply mark the hash element as empty. + if (mf->hash[i] <= subvalue) + mf->hash[i] = EMPTY_HASH_VALUE; + else + mf->hash[i] -= subvalue; + } + + for (uint32_t i = 0; i < mf->sons_count; ++i) { + // Do the same for mf->son. // - // NOTE: Only the first mf->hash_size_sum elements are - // initialized for sure. There may be uninitialized elements - // in mf->son. Since we go through both mf->hash and - // mf->son here in normalization, Valgrind may complain - // that the "if" below depends on uninitialized value. In - // this case it is safe to ignore the warning. See also the - // comments in lz_encoder_init() in lz_encoder.c. - if (hash[i] <= subvalue) - hash[i] = EMPTY_HASH_VALUE; + // NOTE: There may be uninitialized elements in mf->son. + // Valgrind may complain that the "if" below depends on + // an uninitialized value. In this case it is safe to ignore + // the warning. See also the comments in lz_encoder_init() + // in lz_encoder.c. + if (mf->son[i] <= subvalue) + mf->son[i] = EMPTY_HASH_VALUE; else - hash[i] -= subvalue; + mf->son[i] -= subvalue; } // Update offset to match the new locations. @@ -204,14 +198,15 @@ move_pending(lzma_mf *mf) move_pending(mf); \ ret_op; \ } \ - cur = mf_ptr(mf); \ - pos = mf->read_pos + mf->offset + const uint8_t *cur = mf_ptr(mf); \ + const uint32_t pos = mf->read_pos + mf->offset /// Header for find functions. "return 0" indicates that zero matches /// were found. #define header_find(is_bt, len_min) \ - header(is_bt, len_min, return 0) + header(is_bt, len_min, return 0); \ + uint32_t matches_count = 0 /// Header for a loop in a skip function. "continue" tells to skip the rest @@ -268,19 +263,15 @@ hc_find_func( while (true) { const uint32_t delta = pos - cur_match; - const uint8_t *pb; if (depth-- == 0 || delta >= cyclic_size) return matches; - pb = cur - delta; + const uint8_t *const pb = cur - delta; cur_match = son[cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)]; if (pb[len_best] == cur[len_best] && pb[0] == cur[0]) { - uint32_t len = 0; - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + uint32_t len = lzma_memcmplen(pb, cur, 1, len_limit); if (len_best < len) { len_best = len; @@ -313,27 +304,21 @@ do { \ extern uint32_t lzma_mf_hc3_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t delta2, cur_match; - uint32_t len_best = 2; - uint32_t matches_count = 0; - header_find(false, 3); hash_3_calc(); - delta2 = pos - mf->hash[hash_2_value]; - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 2; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen(cur - delta2, cur, + len_best, len_limit); matches[0].len = len_best; matches[0].dist = delta2 - 1; @@ -353,22 +338,18 @@ extern void lzma_mf_hc3_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t cur_match; - if (mf_avail(mf) < 3) { move_pending(mf); continue; } - cur = mf_ptr(mf); - pos = mf->read_pos + mf->offset; + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; hash_3_calc(); - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; @@ -384,25 +365,21 @@ lzma_mf_hc3_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t delta2, delta3, cur_match; - uint32_t len_best = 1; - uint32_t matches_count = 0; - header_find(false, 4); hash_4_calc(); - delta2 = pos - mf->hash[hash_2_value]; - delta3 = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value ] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 1; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { len_best = 2; matches[0].len = 2; @@ -418,9 +395,8 @@ lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches) } if (matches_count != 0) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen(cur - delta2, cur, + len_best, len_limit); matches[matches_count - 1].len = len_best; @@ -441,22 +417,18 @@ extern void lzma_mf_hc4_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t cur_match; - if (mf_avail(mf) < 4) { move_pending(mf); continue; } - cur = mf_ptr(mf); - pos = mf->read_pos + mf->offset; + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; hash_4_calc(); - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; @@ -494,10 +466,6 @@ bt_find_func( uint32_t len1 = 0; while (true) { - uint32_t *pair; - const uint8_t *pb; - uint32_t len; - const uint32_t delta = pos - cur_match; if (depth-- == 0 || delta >= cyclic_size) { *ptr0 = EMPTY_HASH_VALUE; @@ -505,17 +473,15 @@ bt_find_func( return matches; } - pair = son + ((cyclic_pos - delta + uint32_t *const pair = son + ((cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)) << 1); - pb = cur - delta; - len = my_min(len0, len1); + const uint8_t *const pb = cur - delta; + uint32_t len = my_min(len0, len1); if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + len = lzma_memcmplen(pb, cur, len + 1, len_limit); if (len_best < len) { len_best = len; @@ -564,10 +530,6 @@ bt_skip_func( uint32_t len1 = 0; while (true) { - uint32_t *pair; - const uint8_t *pb; - uint32_t len; - const uint32_t delta = pos - cur_match; if (depth-- == 0 || delta >= cyclic_size) { *ptr0 = EMPTY_HASH_VALUE; @@ -575,16 +537,14 @@ bt_skip_func( return; } - pair = son + ((cyclic_pos - delta + uint32_t *pair = son + ((cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)) << 1); - pb = cur - delta; - len = my_min(len0, len1); + const uint8_t *pb = cur - delta; + uint32_t len = my_min(len0, len1); if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + len = lzma_memcmplen(pb, cur, len + 1, len_limit); if (len == len_limit) { *ptr1 = pair[0]; @@ -626,17 +586,11 @@ do { \ extern uint32_t lzma_mf_bt2_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t hash_value; /* hash_2_calc */ - uint32_t cur_match; - uint32_t matches_count = 0; - header_find(true, 2); hash_2_calc(); - cur_match = mf->hash[hash_value]; + const uint32_t cur_match = mf->hash[hash_value]; mf->hash[hash_value] = pos; bt_find(1); @@ -647,16 +601,11 @@ extern void lzma_mf_bt2_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t hash_value; /* hash_2_calc */ - uint32_t cur_match; - header_skip(true, 2); hash_2_calc(); - cur_match = mf->hash[hash_value]; + const uint32_t cur_match = mf->hash[hash_value]; mf->hash[hash_value] = pos; bt_skip(); @@ -670,27 +619,21 @@ lzma_mf_bt2_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_bt3_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t delta2, cur_match; - uint32_t len_best = 2; - uint32_t matches_count = 0; - header_find(true, 3); hash_3_calc(); - delta2 = pos - mf->hash[hash_2_value]; - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 2; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen( + cur, cur - delta2, len_best, len_limit); matches[0].len = len_best; matches[0].dist = delta2 - 1; @@ -710,16 +653,12 @@ extern void lzma_mf_bt3_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t cur_match; - header_skip(true, 3); hash_3_calc(); - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; @@ -735,25 +674,21 @@ lzma_mf_bt3_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t delta2, delta3, cur_match; - uint32_t len_best = 1; - uint32_t matches_count = 0; - header_find(true, 4); hash_4_calc(); - delta2 = pos - mf->hash[hash_2_value]; - delta3 = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 1; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { len_best = 2; matches[0].len = 2; @@ -769,9 +704,8 @@ lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches) } if (matches_count != 0) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen( + cur, cur - delta2, len_best, len_limit); matches[matches_count - 1].len = len_best; @@ -792,16 +726,12 @@ extern void lzma_mf_bt4_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t cur_match; - header_skip(true, 4); hash_4_calc(); - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; diff --git a/Utilities/cmliblzma/liblzma/lzma/fastpos.h b/Utilities/cmliblzma/liblzma/lzma/fastpos.h index 5a834d6..a3feea5 100644 --- a/Utilities/cmliblzma/liblzma/lzma/fastpos.h +++ b/Utilities/cmliblzma/liblzma/lzma/fastpos.h @@ -14,15 +14,15 @@ #ifndef LZMA_FASTPOS_H #define LZMA_FASTPOS_H -// LZMA encodes match distances (positions) by storing the highest two -// bits using a six-bit value [0, 63], and then the missing lower bits. -// Dictionary size is also stored using this encoding in the new .lzma +// LZMA encodes match distances by storing the highest two bits using +// a six-bit value [0, 63], and then the missing lower bits. +// Dictionary size is also stored using this encoding in the .xz // file format header. // // fastpos.h provides a way to quickly find out the correct six-bit // values. The following table gives some examples of this encoding: // -// pos return +// dist return // 0 0 // 1 1 // 2 2 @@ -48,10 +48,10 @@ // Provided functions or macros // ---------------------------- // -// get_pos_slot(pos) is the basic version. get_pos_slot_2(pos) -// assumes that pos >= FULL_DISTANCES, thus the result is at least -// FULL_DISTANCES_BITS * 2. Using get_pos_slot(pos) instead of -// get_pos_slot_2(pos) would give the same result, but get_pos_slot_2(pos) +// get_dist_slot(dist) is the basic version. get_dist_slot_2(dist) +// assumes that dist >= FULL_DISTANCES, thus the result is at least +// FULL_DISTANCES_BITS * 2. Using get_dist_slot(dist) instead of +// get_dist_slot_2(dist) would give the same result, but get_dist_slot_2(dist) // should be tiny bit faster due to the assumption being made. // // @@ -75,16 +75,15 @@ // on all systems I have tried. The size optimized version is sometimes // slightly faster, but sometimes it is a lot slower. -#include "config.h" - #ifdef HAVE_SMALL -# define get_pos_slot(pos) ((pos) <= 4 ? (pos) : get_pos_slot_2(pos)) +# define get_dist_slot(dist) \ + ((dist) <= 4 ? (dist) : get_dist_slot_2(dist)) static inline uint32_t -get_pos_slot_2(uint32_t pos) +get_dist_slot_2(uint32_t dist) { - const uint32_t i = bsr32(pos); - return (i + i) + ((pos >> (i - 1)) & 1); + const uint32_t i = bsr32(dist); + return (i + i) + ((dist >> (i - 1)) & 1); } @@ -101,39 +100,39 @@ extern const uint8_t lzma_fastpos[1 << FASTPOS_BITS]; #define fastpos_limit(extra, n) \ (UINT32_C(1) << (FASTPOS_BITS + fastpos_shift(extra, n))) -#define fastpos_result(pos, extra, n) \ - lzma_fastpos[(pos) >> fastpos_shift(extra, n)] \ +#define fastpos_result(dist, extra, n) \ + lzma_fastpos[(dist) >> fastpos_shift(extra, n)] \ + 2 * fastpos_shift(extra, n) static inline uint32_t -get_pos_slot(uint32_t pos) +get_dist_slot(uint32_t dist) { // If it is small enough, we can pick the result directly from // the precalculated table. - if (pos < fastpos_limit(0, 0)) - return lzma_fastpos[pos]; + if (dist < fastpos_limit(0, 0)) + return lzma_fastpos[dist]; - if (pos < fastpos_limit(0, 1)) - return fastpos_result(pos, 0, 1); + if (dist < fastpos_limit(0, 1)) + return fastpos_result(dist, 0, 1); - return fastpos_result(pos, 0, 2); + return fastpos_result(dist, 0, 2); } #ifdef FULL_DISTANCES_BITS static inline uint32_t -get_pos_slot_2(uint32_t pos) +get_dist_slot_2(uint32_t dist) { - assert(pos >= FULL_DISTANCES); + assert(dist >= FULL_DISTANCES); - if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 0)) - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 0); + if (dist < fastpos_limit(FULL_DISTANCES_BITS - 1, 0)) + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 0); - if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 1)) - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 1); + if (dist < fastpos_limit(FULL_DISTANCES_BITS - 1, 1)) + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 1); - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 2); + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 2); } #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c index bd2a737..878c870 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c @@ -16,7 +16,7 @@ #include "lzma_decoder.h" -struct lzma_coder_s { +typedef struct { enum sequence { SEQ_CONTROL, SEQ_UNCOMPRESSED_1, @@ -50,14 +50,16 @@ struct lzma_coder_s { bool need_dictionary_reset; lzma_options_lzma options; -}; +} lzma_lzma2_coder; static lzma_ret -lzma2_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dict, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lzma2_decode(void *coder_ptr, lzma_dict *restrict dict, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size) { + lzma_lzma2_coder *restrict coder = coder_ptr; + // With SEQ_LZMA it is possible that no new input is needed to do // some progress. The rest of the sequences assume that there is // at least one byte of input. @@ -209,8 +211,10 @@ lzma2_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dict, static void -lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +lzma2_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_lzma2_coder *coder = coder_ptr; + assert(coder->lzma.end == NULL); lzma_free(coder->lzma.coder, allocator); @@ -221,34 +225,36 @@ lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lzma2_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma2_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options) { - const lzma_options_lzma *options = opt; - - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (lz->coder == NULL) + lzma_lzma2_coder *coder = lz->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_lzma2_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + lz->coder = coder; lz->code = &lzma2_decode; lz->end = &lzma2_decoder_end; - lz->coder->lzma = LZMA_LZ_DECODER_INIT; + coder->lzma = LZMA_LZ_DECODER_INIT; } - lz->coder->sequence = SEQ_CONTROL; - lz->coder->need_properties = true; - lz->coder->need_dictionary_reset = options->preset_dict == NULL + const lzma_options_lzma *options = opt; + + coder->sequence = SEQ_CONTROL; + coder->need_properties = true; + coder->need_dictionary_reset = options->preset_dict == NULL || options->preset_dict_size == 0; - return lzma_lzma_decoder_create(&lz->coder->lzma, + return lzma_lzma_decoder_create(&coder->lzma, allocator, options, lz_options); } extern lzma_ret -lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma2_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { // LZMA2 can only be the last filter in the chain. This is enforced @@ -263,17 +269,15 @@ lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern uint64_t lzma_lzma2_decoder_memusage(const void *options) { - return sizeof(lzma_coder) + return sizeof(lzma_lzma2_coder) + lzma_lzma_decoder_memusage_nocheck(options); } extern lzma_ret -lzma_lzma2_props_decode(void **options, lzma_allocator *allocator, +lzma_lzma2_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_lzma *opt; - if (props_size != 1) return LZMA_OPTIONS_ERROR; @@ -285,7 +289,8 @@ lzma_lzma2_props_decode(void **options, lzma_allocator *allocator, if (props[0] > 40) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_lzma), allocator); + lzma_options_lzma *opt = lzma_alloc( + sizeof(lzma_options_lzma), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h index fac4ac4..ef2dcbf 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h @@ -17,12 +17,13 @@ #include "common.h" extern lzma_ret lzma_lzma2_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma2_decoder_memusage(const void *options); extern lzma_ret lzma_lzma2_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c index a3651a7..63588ee 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c @@ -17,7 +17,7 @@ #include "lzma2_encoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INIT, SEQ_LZMA_ENCODE, @@ -27,7 +27,7 @@ struct lzma_coder_s { } sequence; /// LZMA encoder - lzma_coder *lzma; + void *lzma; /// LZMA options currently in use. lzma_options_lzma opt_cur; @@ -48,20 +48,19 @@ struct lzma_coder_s { /// Buffer to hold the chunk header and LZMA compressed data uint8_t buf[LZMA2_HEADER_MAX + LZMA2_CHUNK_MAX]; -}; +} lzma_lzma2_coder; static void -lzma2_header_lzma(lzma_coder *coder) +lzma2_header_lzma(lzma_lzma2_coder *coder) { - size_t pos; - size_t size; - assert(coder->uncompressed_size > 0); assert(coder->uncompressed_size <= LZMA2_UNCOMPRESSED_MAX); assert(coder->compressed_size > 0); assert(coder->compressed_size <= LZMA2_CHUNK_MAX); + size_t pos; + if (coder->need_properties) { pos = 0; @@ -82,7 +81,7 @@ lzma2_header_lzma(lzma_coder *coder) coder->buf_pos = pos; // Uncompressed size - size = coder->uncompressed_size - 1; + size_t size = coder->uncompressed_size - 1; coder->buf[pos++] += size >> 16; coder->buf[pos++] = (size >> 8) & 0xFF; coder->buf[pos++] = size & 0xFF; @@ -109,7 +108,7 @@ lzma2_header_lzma(lzma_coder *coder) static void -lzma2_header_uncompressed(lzma_coder *coder) +lzma2_header_uncompressed(lzma_lzma2_coder *coder) { assert(coder->uncompressed_size > 0); assert(coder->uncompressed_size <= LZMA2_CHUNK_MAX); @@ -134,10 +133,12 @@ lzma2_header_uncompressed(lzma_coder *coder) static lzma_ret -lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma2_encode(void *coder_ptr, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { + lzma_lzma2_coder *restrict coder = coder_ptr; + while (*out_pos < out_size) switch (coder->sequence) { case SEQ_INIT: @@ -163,9 +164,6 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Fall through case SEQ_LZMA_ENCODE: { - uint32_t read_start; - lzma_ret ret; - // Calculate how much more uncompressed data this chunk // could accept. const uint32_t left = LZMA2_UNCOMPRESSED_MAX @@ -186,10 +184,10 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Save the start position so that we can update // coder->uncompressed_size. - read_start = mf->read_pos - mf->read_ahead; + const uint32_t read_start = mf->read_pos - mf->read_ahead; // Call the LZMA encoder until the chunk is finished. - ret = lzma_lzma_encode(coder->lzma, mf, + const lzma_ret ret = lzma_lzma_encode(coder->lzma, mf, coder->buf + LZMA2_HEADER_MAX, &coder->compressed_size, LZMA2_CHUNK_MAX, limit); @@ -266,8 +264,9 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static void -lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +lzma2_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_lzma2_coder *coder = coder_ptr; lzma_free(coder->lzma, allocator); lzma_free(coder, allocator); return; @@ -275,9 +274,9 @@ lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) +lzma2_encoder_options_update(void *coder_ptr, const lzma_filter *filter) { - lzma_options_lzma *opt; + lzma_lzma2_coder *coder = coder_ptr; // New options can be set only when there is no incomplete chunk. // This is the case at the beginning of the raw stream and right @@ -287,7 +286,7 @@ lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) // Look if there are new options. At least for now, // only lc/lp/pb can be changed. - opt = filter->options; + const lzma_options_lzma *opt = filter->options; if (coder->opt_cur.lc != opt->lc || coder->opt_cur.lp != opt->lp || coder->opt_cur.pb != opt->pb) { // Validate the options. @@ -310,36 +309,38 @@ lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) static lzma_ret -lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, +lzma2_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { if (options == NULL) return LZMA_PROG_ERROR; - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (lz->coder == NULL) + lzma_lzma2_coder *coder = lz->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_lzma2_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + lz->coder = coder; lz->code = &lzma2_encode; lz->end = &lzma2_encoder_end; lz->options_update = &lzma2_encoder_options_update; - lz->coder->lzma = NULL; + coder->lzma = NULL; } - lz->coder->opt_cur = *(const lzma_options_lzma *)(options); + coder->opt_cur = *(const lzma_options_lzma *)(options); - lz->coder->sequence = SEQ_INIT; - lz->coder->need_properties = true; - lz->coder->need_state_reset = false; - lz->coder->need_dictionary_reset - = lz->coder->opt_cur.preset_dict == NULL - || lz->coder->opt_cur.preset_dict_size == 0; + coder->sequence = SEQ_INIT; + coder->need_properties = true; + coder->need_state_reset = false; + coder->need_dictionary_reset + = coder->opt_cur.preset_dict == NULL + || coder->opt_cur.preset_dict_size == 0; // Initialize LZMA encoder - return_if_error(lzma_lzma_encoder_create(&lz->coder->lzma, allocator, - &lz->coder->opt_cur, lz_options)); + return_if_error(lzma_lzma_encoder_create(&coder->lzma, allocator, + &coder->opt_cur, lz_options)); // Make sure that we will always have enough history available in // case we need to use uncompressed chunks. They are used when the @@ -355,7 +356,7 @@ lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma2_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma2_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { return lzma_lz_encoder_init( @@ -370,7 +371,7 @@ lzma_lzma2_encoder_memusage(const void *options) if (lzma_mem == UINT64_MAX) return UINT64_MAX; - return sizeof(lzma_coder) + lzma_mem; + return sizeof(lzma_lzma2_coder) + lzma_mem; } @@ -393,7 +394,17 @@ lzma_lzma2_props_encode(const void *options, uint8_t *out) if (d == UINT32_MAX) out[0] = 40; else - out[0] = get_pos_slot(d + 1) - 24; + out[0] = get_dist_slot(d + 1) - 24; return LZMA_OK; } + + +extern uint64_t +lzma_lzma2_block_size(const void *options) +{ + const lzma_options_lzma *const opt = options; + + // Use at least 1 MiB to keep compression ratio better. + return my_max((uint64_t)(opt->dict_size) * 3, UINT64_C(1) << 20); +} diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h index ca19ef4..515f183 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h @@ -31,11 +31,13 @@ extern lzma_ret lzma_lzma2_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); extern uint64_t lzma_lzma2_encoder_memusage(const void *options); extern lzma_ret lzma_lzma2_props_encode(const void *options, uint8_t *out); +extern uint64_t lzma_lzma2_block_size(const void *options); + #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_common.h b/Utilities/cmliblzma/liblzma/lzma/lzma_common.h index 36267dc..09efd38 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_common.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_common.h @@ -129,15 +129,12 @@ static inline void literal_init(probability (*probs)[LITERAL_CODER_SIZE], uint32_t lc, uint32_t lp) { - uint32_t coders; - uint32_t i, j; - assert(lc + lp <= LZMA_LCLP_MAX); - coders = 1U << (lc + lp); + const uint32_t coders = 1U << (lc + lp); - for (i = 0; i < coders; ++i) - for (j = 0; j < LITERAL_CODER_SIZE; ++j) + for (uint32_t i = 0; i < coders; ++i) + for (uint32_t j = 0; j < LITERAL_CODER_SIZE; ++j) bit_reset(probs[i][j]); return; @@ -174,53 +171,54 @@ literal_init(probability (*probs)[LITERAL_CODER_SIZE], // Match distance // //////////////////// -// Different set of probabilities is used for match distances that have very +// Different sets of probabilities are used for match distances that have very // short match length: Lengths of 2, 3, and 4 bytes have a separate set of // probabilities for each length. The matches with longer length use a shared // set of probabilities. -#define LEN_TO_POS_STATES 4 +#define DIST_STATES 4 // Macro to get the index of the appropriate probability array. -#define get_len_to_pos_state(len) \ - ((len) < LEN_TO_POS_STATES + MATCH_LEN_MIN \ +#define get_dist_state(len) \ + ((len) < DIST_STATES + MATCH_LEN_MIN \ ? (len) - MATCH_LEN_MIN \ - : LEN_TO_POS_STATES - 1) + : DIST_STATES - 1) -// The highest two bits of a match distance (pos slot) are encoded using six -// bits. See fastpos.h for more explanation. -#define POS_SLOT_BITS 6 -#define POS_SLOTS (1 << POS_SLOT_BITS) +// The highest two bits of a match distance (distance slot) are encoded +// using six bits. See fastpos.h for more explanation. +#define DIST_SLOT_BITS 6 +#define DIST_SLOTS (1 << DIST_SLOT_BITS) // Match distances up to 127 are fully encoded using probabilities. Since -// the highest two bits (pos slot) are always encoded using six bits, the -// distances 0-3 don't need any additional bits to encode, since the pos -// slot itself is the same as the actual distance. START_POS_MODEL_INDEX -// indicates the first pos slot where at least one additional bit is needed. -#define START_POS_MODEL_INDEX 4 +// the highest two bits (distance slot) are always encoded using six bits, +// the distances 0-3 don't need any additional bits to encode, since the +// distance slot itself is the same as the actual distance. DIST_MODEL_START +// indicates the first distance slot where at least one additional bit is +// needed. +#define DIST_MODEL_START 4 // Match distances greater than 127 are encoded in three pieces: -// - pos slot: the highest two bits +// - distance slot: the highest two bits // - direct bits: 2-26 bits below the highest two bits // - alignment bits: four lowest bits // // Direct bits don't use any probabilities. // -// The pos slot value of 14 is for distances 128-191 (see the table in +// The distance slot value of 14 is for distances 128-191 (see the table in // fastpos.h to understand why). -#define END_POS_MODEL_INDEX 14 +#define DIST_MODEL_END 14 -// Pos slots that indicate a distance <= 127. -#define FULL_DISTANCES_BITS (END_POS_MODEL_INDEX / 2) +// Distance slots that indicate a distance <= 127. +#define FULL_DISTANCES_BITS (DIST_MODEL_END / 2) #define FULL_DISTANCES (1 << FULL_DISTANCES_BITS) // For match distances greater than 127, only the highest two bits and the // lowest four bits (alignment) is encoded using probabilities. #define ALIGN_BITS 4 -#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS) -#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1) +#define ALIGN_SIZE (1 << ALIGN_BITS) +#define ALIGN_MASK (ALIGN_SIZE - 1) // LZMA remembers the four most recent match distances. Reusing these distances // tends to take less space than re-encoding the actual distance value. -#define REP_DISTANCES 4 +#define REPS 4 #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c index 3c0f393..d0f29b7 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c @@ -16,6 +16,12 @@ #include "lzma_decoder.h" #include "range_decoder.h" +// The macros unroll loops with switch statements. +// Silence warnings about missing fall-through comments. +#if TUKLIB_GNUC_REQ(7, 0) +# pragma GCC diagnostic ignored "-Wimplicit-fallthrough" +#endif + #ifdef HAVE_SMALL @@ -114,33 +120,33 @@ do { \ case seq ## _CHOICE: \ rc_if_0(ld.choice, seq ## _CHOICE) { \ rc_update_0(ld.choice); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW0); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW1); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW2); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW0); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW1); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW2); \ target = symbol - LEN_LOW_SYMBOLS + MATCH_LEN_MIN; \ } else { \ rc_update_1(ld.choice); \ case seq ## _CHOICE2: \ rc_if_0(ld.choice2, seq ## _CHOICE2) { \ rc_update_0(ld.choice2); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID0); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID1); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID2); \ target = symbol - LEN_MID_SYMBOLS \ + MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \ } else { \ rc_update_1(ld.choice2); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH0); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH1); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH2); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH3); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH4); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH5); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH6); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH7); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH0); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH1); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH2); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH3); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH4); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH5); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH6); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH7); \ target = symbol - LEN_HIGH_SYMBOLS \ + MATCH_LEN_MIN \ + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ @@ -161,7 +167,7 @@ typedef struct { } lzma_length_decoder; -struct lzma_coder_s { +typedef struct { /////////////////// // Probabilities // /////////////////// @@ -193,15 +199,15 @@ struct lzma_coder_s { /// Probability tree for the highest two bits of the match distance. /// There is a separate probability tree for match lengths of /// 2 (i.e. MATCH_LEN_MIN), 3, 4, and [5, 273]. - probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; + probability dist_slot[DIST_STATES][DIST_SLOTS]; /// Probability trees for additional bits for match distance when the /// distance is in the range [4, 127]. - probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; + probability pos_special[FULL_DISTANCES - DIST_MODEL_END]; /// Probability tree for the lowest four bits of a match distance /// that is equal to or greater than 128. - probability pos_align[ALIGN_TABLE_SIZE]; + probability pos_align[ALIGN_SIZE]; /// Length of a normal match lzma_length_decoder match_len_decoder; @@ -245,8 +251,8 @@ struct lzma_coder_s { SEQ_LITERAL_WRITE, SEQ_IS_REP, seq_len(SEQ_MATCH_LEN), - seq_6(SEQ_POS_SLOT), - SEQ_POS_MODEL, + seq_6(SEQ_DIST_SLOT), + SEQ_DIST_MODEL, SEQ_DIRECT, seq_4(SEQ_ALIGN), SEQ_EOPM, @@ -277,14 +283,27 @@ struct lzma_coder_s { /// If decoding a literal: match byte. /// If decoding a match: length of the match. uint32_t len; -}; +} lzma_lzma1_decoder; static lzma_ret -lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, - const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size) +lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, + const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) { + lzma_lzma1_decoder *restrict coder = coder_ptr; + + //////////////////// + // Initialization // + //////////////////// + + { + const lzma_ret ret = rc_read_init( + &coder->rc, in, in_pos, in_size); + if (ret != LZMA_STREAM_END) + return ret; + } + /////////////// // Variables // /////////////// @@ -331,16 +350,6 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, if (no_eopm && coder->uncompressed_size < dict.limit - dict.pos) dict.limit = dict.pos + (size_t)(coder->uncompressed_size); - //////////////////// - // Initialization // - //////////////////// - - if (!rc_read_init(&coder->rc, in, in_pos, in_size)) - return LZMA_OK; - - rc = coder->rc; - rc_in_pos = *in_pos; - // The main decoder loop. The "switch" is used to restart the decoder at // correct location. Once restarted, the "switch" is no longer used. switch (coder->sequence) @@ -356,21 +365,6 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, break; rc_if_0(coder->is_match[state][pos_state], SEQ_IS_MATCH) { - static const lzma_lzma_state next_state[] = { - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_MATCH_LIT_LIT, - STATE_REP_LIT_LIT, - STATE_SHORTREP_LIT_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT, - STATE_SHORTREP_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT - }; - rc_update_0(coder->is_match[state][pos_state]); // It's a literal i.e. a single 8-bit byte. @@ -388,21 +382,16 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, rc_bit(probs[symbol], , , SEQ_LITERAL); } while (symbol < (1 << 8)); #else - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL0); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL1); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL2); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL3); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL4); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL5); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL6); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL7); + rc_bit_case(probs[symbol], , , SEQ_LITERAL0); + rc_bit_case(probs[symbol], , , SEQ_LITERAL1); + rc_bit_case(probs[symbol], , , SEQ_LITERAL2); + rc_bit_case(probs[symbol], , , SEQ_LITERAL3); + rc_bit_case(probs[symbol], , , SEQ_LITERAL4); + rc_bit_case(probs[symbol], , , SEQ_LITERAL5); + rc_bit_case(probs[symbol], , , SEQ_LITERAL6); + rc_bit_case(probs[symbol], , , SEQ_LITERAL7); #endif } else { -#ifndef HAVE_SMALL - uint32_t match_bit; - uint32_t subcoder_index; -#endif - // Decode literal with match byte. // // We store the byte we compare against @@ -441,6 +430,8 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, } while (symbol < (1 << 8)); #else // Unroll the loop. + uint32_t match_bit; + uint32_t subcoder_index; # define d(seq) \ case seq: \ @@ -474,6 +465,20 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // Use a lookup table to update to literal state, // since compared to other state updates, this would // need two branches. + static const lzma_lzma_state next_state[] = { + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_MATCH_LIT_LIT, + STATE_REP_LIT_LIT, + STATE_SHORTREP_LIT_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT, + STATE_SHORTREP_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT + }; state = next_state[state]; case SEQ_LITERAL_WRITE: @@ -509,28 +514,28 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // Prepare to decode the highest two bits of the // match distance. - probs = coder->pos_slot[get_len_to_pos_state(len)]; + probs = coder->dist_slot[get_dist_state(len)]; symbol = 1; #ifdef HAVE_SMALL - case SEQ_POS_SLOT: + case SEQ_DIST_SLOT: do { - rc_bit(probs[symbol], , , SEQ_POS_SLOT); - } while (symbol < POS_SLOTS); + rc_bit(probs[symbol], , , SEQ_DIST_SLOT); + } while (symbol < DIST_SLOTS); #else - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT0); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT1); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT2); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT3); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT4); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT5); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT0); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT1); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT2); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT3); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT4); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT5); #endif // Get rid of the highest bit that was needed for // indexing of the probability array. - symbol -= POS_SLOTS; + symbol -= DIST_SLOTS; assert(symbol <= 63); - if (symbol < START_POS_MODEL_INDEX) { + if (symbol < DIST_MODEL_START) { // Match distances [0, 3] have only two bits. rep0 = symbol; } else { @@ -540,7 +545,7 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, assert(limit >= 1 && limit <= 30); rep0 = 2 + (symbol & 1); - if (symbol < END_POS_MODEL_INDEX) { + if (symbol < DIST_MODEL_END) { // Prepare to decode the low bits for // a distance of [4, 127]. assert(limit <= 5); @@ -560,38 +565,38 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, - symbol - 1; symbol = 1; offset = 0; - case SEQ_POS_MODEL: + case SEQ_DIST_MODEL: #ifdef HAVE_SMALL do { rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); } while (++offset < limit); #else switch (limit) { case 5: assert(offset == 0); - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 4: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 3: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 2: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 1: @@ -601,9 +606,9 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // rc_bit_last() here to omit // the unneeded updating of // "symbol". - rc_bit_last(probs[symbol], 0, + rc_bit_last(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); } #endif } else { @@ -635,19 +640,19 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, } while (++offset < ALIGN_BITS); #else case SEQ_ALIGN0: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 1, SEQ_ALIGN0); case SEQ_ALIGN1: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 2, SEQ_ALIGN1); case SEQ_ALIGN2: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 4, SEQ_ALIGN2); case SEQ_ALIGN3: - // Like in SEQ_POS_MODEL, we don't + // Like in SEQ_DIST_MODEL, we don't // need "symbol" for anything else // than indexing the probability array. - rc_bit_last(coder->pos_align[symbol], 0, + rc_bit_last(coder->pos_align[symbol], , rep0 += 8, SEQ_ALIGN3); #endif @@ -732,11 +737,9 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // is stored to rep0 and rep1, rep2 and rep3 // are updated accordingly. rc_if_0(coder->is_rep1[state], SEQ_IS_REP1) { - uint32_t distance; - rc_update_0(coder->is_rep1[state]); - distance = rep1; + const uint32_t distance = rep1; rep1 = rep0; rep0 = distance; @@ -745,23 +748,19 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, case SEQ_IS_REP2: rc_if_0(coder->is_rep2[state], SEQ_IS_REP2) { - uint32_t distance; - rc_update_0(coder->is_rep2[ state]); - distance = rep2; + const uint32_t distance = rep2; rep2 = rep1; rep1 = rep0; rep0 = distance; } else { - uint32_t distance; - rc_update_1(coder->is_rep2[ state]); - distance = rep3; + const uint32_t distance = rep3; rep3 = rep2; rep2 = rep1; rep1 = rep0; @@ -849,26 +848,17 @@ out: static void -lzma_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) +lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) { + lzma_lzma1_decoder *coder = coder_ptr; coder->uncompressed_size = uncompressed_size; } -/* -extern void -lzma_lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) -{ - // This is hack. - (*(lzma_coder **)(coder))->uncompressed_size = uncompressed_size; -} -*/ static void -lzma_decoder_reset(lzma_coder *coder, const void *opt) +lzma_decoder_reset(void *coder_ptr, const void *opt) { - uint32_t i, j, pos_state; - uint32_t num_pos_states; - + lzma_lzma1_decoder *coder = coder_ptr; const lzma_options_lzma *options = opt; // NOTE: We assume that lc/lp/pb are valid since they were @@ -895,8 +885,8 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) rc_reset(coder->rc); // Bit and bittree decoders - for (i = 0; i < STATES; ++i) { - for (j = 0; j <= coder->pos_mask; ++j) { + for (uint32_t i = 0; i < STATES; ++i) { + for (uint32_t j = 0; j <= coder->pos_mask; ++j) { bit_reset(coder->is_match[i][j]); bit_reset(coder->is_rep0_long[i][j]); } @@ -907,22 +897,22 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) bit_reset(coder->is_rep2[i]); } - for (i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); + for (uint32_t i = 0; i < DIST_STATES; ++i) + bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS); - for (i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) + for (uint32_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i) bit_reset(coder->pos_special[i]); bittree_reset(coder->pos_align, ALIGN_BITS); // Len decoders (also bit/bittree) - num_pos_states = 1U << options->pb; + const uint32_t num_pos_states = 1U << options->pb; bit_reset(coder->match_len_decoder.choice); bit_reset(coder->match_len_decoder.choice2); bit_reset(coder->rep_len_decoder.choice); bit_reset(coder->rep_len_decoder.choice2); - for (pos_state = 0; pos_state < num_pos_states; ++pos_state) { + for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { bittree_reset(coder->match_len_decoder.low[pos_state], LEN_LOW_BITS); bittree_reset(coder->match_len_decoder.mid[pos_state], @@ -949,13 +939,11 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) extern lzma_ret -lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma_lzma_decoder_create(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options) { - const lzma_options_lzma *options = opt; - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); + lz->coder = lzma_alloc(sizeof(lzma_lzma1_decoder), allocator); if (lz->coder == NULL) return LZMA_MEM_ERROR; @@ -966,6 +954,7 @@ lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, // All dictionary sizes are OK here. LZ decoder will take care of // the special cases. + const lzma_options_lzma *options = opt; lz_options->dict_size = options->dict_size; lz_options->preset_dict = options->preset_dict; lz_options->preset_dict_size = options->preset_dict_size; @@ -978,7 +967,7 @@ lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, /// initialization (lzma_lzma_decoder_init() passes function pointer to /// the LZ initialization). static lzma_ret -lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { if (!is_lclppb_valid(options)) @@ -995,7 +984,7 @@ lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { // LZMA can only be the last filter in the chain. This is enforced @@ -1027,7 +1016,8 @@ extern uint64_t lzma_lzma_decoder_memusage_nocheck(const void *options) { const lzma_options_lzma *const opt = options; - return sizeof(lzma_coder) + lzma_lz_decoder_memusage(opt->dict_size); + return sizeof(lzma_lzma1_decoder) + + lzma_lz_decoder_memusage(opt->dict_size); } @@ -1042,15 +1032,14 @@ lzma_lzma_decoder_memusage(const void *options) extern lzma_ret -lzma_lzma_props_decode(void **options, lzma_allocator *allocator, +lzma_lzma_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_lzma *opt; - if (props_size != 5) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_lzma), allocator); + lzma_options_lzma *opt + = lzma_alloc(sizeof(lzma_options_lzma), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h index a463a76..fa8ecb2 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h @@ -19,12 +19,13 @@ /// Allocates and initializes LZMA decoder extern lzma_ret lzma_lzma_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma_decoder_memusage(const void *options); extern lzma_ret lzma_lzma_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); @@ -40,7 +41,7 @@ extern bool lzma_lzma_lclppb_decode( /// Allocate and setup function pointers only. This is used by LZMA1 and /// LZMA2 decoders. extern lzma_ret lzma_lzma_decoder_create( - lzma_lz_decoder *lz, lzma_allocator *allocator, + lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options); /// Gets memory usage without validating lc/lp/pb. This is used by LZMA2 diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c index e8738f4..ba9ce69 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c @@ -28,14 +28,11 @@ literal_matched(lzma_range_encoder *rc, probability *subcoder, symbol += UINT32_C(1) << 8; do { - uint32_t match_bit; - uint32_t subcoder_index; - uint32_t bit; - match_byte <<= 1; - match_bit = match_byte & offset; - subcoder_index = offset + match_bit + (symbol >> 8); - bit = (symbol >> 7) & 1; + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; rc_bit(rc, &subcoder[subcoder_index], bit); symbol <<= 1; @@ -46,7 +43,7 @@ literal_matched(lzma_range_encoder *rc, probability *subcoder, static inline void -literal(lzma_coder *coder, lzma_mf *mf, uint32_t position) +literal(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t position) { // Locate the literal byte to be encoded and the subcoder. const uint8_t cur_byte = mf->buffer[ @@ -80,19 +77,16 @@ literal(lzma_coder *coder, lzma_mf *mf, uint32_t position) static void length_update_prices(lzma_length_encoder *lc, const uint32_t pos_state) { - uint32_t a0, a1, b0, b1; - uint32_t *prices; - uint32_t i; - const uint32_t table_size = lc->table_size; lc->counters[pos_state] = table_size; - a0 = rc_bit_0_price(lc->choice); - a1 = rc_bit_1_price(lc->choice); - b0 = a1 + rc_bit_0_price(lc->choice2); - b1 = a1 + rc_bit_1_price(lc->choice2); - prices = lc->prices[pos_state]; + const uint32_t a0 = rc_bit_0_price(lc->choice); + const uint32_t a1 = rc_bit_1_price(lc->choice); + const uint32_t b0 = a1 + rc_bit_0_price(lc->choice2); + const uint32_t b1 = a1 + rc_bit_1_price(lc->choice2); + uint32_t *const prices = lc->prices[pos_state]; + uint32_t i; for (i = 0; i < table_size && i < LEN_LOW_SYMBOLS; ++i) prices[i] = a0 + rc_bittree_price(lc->low[pos_state], LEN_LOW_BITS, i); @@ -146,39 +140,36 @@ length(lzma_range_encoder *rc, lzma_length_encoder *lc, /////////// static inline void -match(lzma_coder *coder, const uint32_t pos_state, +match(lzma_lzma1_encoder *coder, const uint32_t pos_state, const uint32_t distance, const uint32_t len) { - uint32_t pos_slot; - uint32_t len_to_pos_state; - update_match(coder->state); length(&coder->rc, &coder->match_len_encoder, pos_state, len, coder->fast_mode); - pos_slot = get_pos_slot(distance); - len_to_pos_state = get_len_to_pos_state(len); - rc_bittree(&coder->rc, coder->pos_slot[len_to_pos_state], - POS_SLOT_BITS, pos_slot); + const uint32_t dist_slot = get_dist_slot(distance); + const uint32_t dist_state = get_dist_state(len); + rc_bittree(&coder->rc, coder->dist_slot[dist_state], + DIST_SLOT_BITS, dist_slot); - if (pos_slot >= START_POS_MODEL_INDEX) { - const uint32_t footer_bits = (pos_slot >> 1) - 1; - const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; - const uint32_t pos_reduced = distance - base; + if (dist_slot >= DIST_MODEL_START) { + const uint32_t footer_bits = (dist_slot >> 1) - 1; + const uint32_t base = (2 | (dist_slot & 1)) << footer_bits; + const uint32_t dist_reduced = distance - base; - if (pos_slot < END_POS_MODEL_INDEX) { - // Careful here: base - pos_slot - 1 can be -1, but + if (dist_slot < DIST_MODEL_END) { + // Careful here: base - dist_slot - 1 can be -1, but // rc_bittree_reverse starts at probs[1], not probs[0]. rc_bittree_reverse(&coder->rc, - coder->pos_special + base - pos_slot - 1, - footer_bits, pos_reduced); + coder->dist_special + base - dist_slot - 1, + footer_bits, dist_reduced); } else { - rc_direct(&coder->rc, pos_reduced >> ALIGN_BITS, + rc_direct(&coder->rc, dist_reduced >> ALIGN_BITS, footer_bits - ALIGN_BITS); rc_bittree_reverse( - &coder->rc, coder->pos_align, - ALIGN_BITS, pos_reduced & ALIGN_MASK); + &coder->rc, coder->dist_align, + ALIGN_BITS, dist_reduced & ALIGN_MASK); ++coder->align_price_count; } } @@ -196,7 +187,7 @@ match(lzma_coder *coder, const uint32_t pos_state, //////////////////// static inline void -rep_match(lzma_coder *coder, const uint32_t pos_state, +rep_match(lzma_lzma1_encoder *coder, const uint32_t pos_state, const uint32_t rep, const uint32_t len) { if (rep == 0) { @@ -240,7 +231,7 @@ rep_match(lzma_coder *coder, const uint32_t pos_state, ////////// static void -encode_symbol(lzma_coder *coder, lzma_mf *mf, +encode_symbol(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t back, uint32_t len, uint32_t position) { const uint32_t pos_state = position & coder->pos_mask; @@ -256,7 +247,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); - if (back < REP_DISTANCES) { + if (back < REPS) { // It's a repeated match i.e. the same distance // has been used earlier. rc_bit(&coder->rc, &coder->is_rep[coder->state], 1); @@ -264,7 +255,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, } else { // Normal match rc_bit(&coder->rc, &coder->is_rep[coder->state], 0); - match(coder, pos_state, back - REP_DISTANCES, len); + match(coder, pos_state, back - REPS, len); } } @@ -274,7 +265,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, static bool -encode_init(lzma_coder *coder, lzma_mf *mf) +encode_init(lzma_lzma1_encoder *coder, lzma_mf *mf) { assert(mf_position(mf) == 0); @@ -302,7 +293,7 @@ encode_init(lzma_coder *coder, lzma_mf *mf) static void -encode_eopm(lzma_coder *coder, uint32_t position) +encode_eopm(lzma_lzma1_encoder *coder, uint32_t position) { const uint32_t pos_state = position & coder->pos_mask; rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); @@ -318,23 +309,18 @@ encode_eopm(lzma_coder *coder, uint32_t position) extern lzma_ret -lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma_lzma_encode(lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, uint32_t limit) { - uint32_t position; - // Initialize the stream if no data has been encoded yet. if (!coder->is_initialized && !encode_init(coder, mf)) return LZMA_OK; // Get the lowest bits of the uncompressed offset from the LZ layer. - position = mf_position(mf); + uint32_t position = mf_position(mf); while (true) { - uint32_t len; - uint32_t back; - // Encode pending bits, if any. Calling this before encoding // the next symbol is needed only with plain LZMA, since // LZMA2 always provides big enough buffer to flush @@ -367,12 +353,14 @@ lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Get optimal match (repeat position and length). // Value ranges for pos: - // - [0, REP_DISTANCES): repeated match - // - [REP_DISTANCES, UINT32_MAX): - // match at (pos - REP_DISTANCES) + // - [0, REPS): repeated match + // - [REPS, UINT32_MAX): + // match at (pos - REPS) // - UINT32_MAX: not a match but a literal // Value ranges for len: // - [MATCH_LEN_MIN, MATCH_LEN_MAX] + uint32_t len; + uint32_t back; if (coder->fast_mode) lzma_lzma_optimum_fast(coder, mf, &back, &len); @@ -414,8 +402,8 @@ lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static lzma_ret -lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma_encode(void *coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { // Plain LZMA has no support for sync-flushing. @@ -465,12 +453,10 @@ static void length_encoder_reset(lzma_length_encoder *lencoder, const uint32_t num_pos_states, const bool fast_mode) { - size_t pos_state; - bit_reset(lencoder->choice); bit_reset(lencoder->choice2); - for (pos_state = 0; pos_state < num_pos_states; ++pos_state) { + for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS); bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS); } @@ -478,7 +464,7 @@ length_encoder_reset(lzma_length_encoder *lencoder, bittree_reset(lencoder->high, LEN_HIGH_BITS); if (!fast_mode) - for (pos_state = 0; pos_state < num_pos_states; + for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) length_update_prices(lencoder, pos_state); @@ -487,10 +473,9 @@ length_encoder_reset(lzma_length_encoder *lencoder, extern lzma_ret -lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) +lzma_lzma_encoder_reset(lzma_lzma1_encoder *coder, + const lzma_options_lzma *options) { - size_t i, j; - if (!is_options_valid(options)) return LZMA_OPTIONS_ERROR; @@ -503,14 +488,14 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) // State coder->state = STATE_LIT_LIT; - for (i = 0; i < REP_DISTANCES; ++i) + for (size_t i = 0; i < REPS; ++i) coder->reps[i] = 0; literal_init(coder->literal, options->lc, options->lp); // Bit encoders - for (i = 0; i < STATES; ++i) { - for (j = 0; j <= coder->pos_mask; ++j) { + for (size_t i = 0; i < STATES; ++i) { + for (size_t j = 0; j <= coder->pos_mask; ++j) { bit_reset(coder->is_match[i][j]); bit_reset(coder->is_rep0_long[i][j]); } @@ -521,14 +506,14 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) bit_reset(coder->is_rep2[i]); } - for (i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) - bit_reset(coder->pos_special[i]); + for (size_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i) + bit_reset(coder->dist_special[i]); // Bit tree encoders - for (i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); + for (size_t i = 0; i < DIST_STATES; ++i) + bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS); - bittree_reset(coder->pos_align, ALIGN_BITS); + bittree_reset(coder->dist_align, ALIGN_BITS); // Length encoders length_encoder_reset(&coder->match_len_encoder, @@ -561,20 +546,18 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) extern lzma_ret -lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, +lzma_lzma_encoder_create(void **coder_ptr, + const lzma_allocator *allocator, const lzma_options_lzma *options, lzma_lz_options *lz_options) { - lzma_coder *coder; - uint32_t log_size = 0; - - // Allocate lzma_coder if it wasn't already allocated. + // Allocate lzma_lzma1_encoder if it wasn't already allocated. if (*coder_ptr == NULL) { - *coder_ptr = lzma_alloc(sizeof(lzma_coder), allocator); + *coder_ptr = lzma_alloc(sizeof(lzma_lzma1_encoder), allocator); if (*coder_ptr == NULL) return LZMA_MEM_ERROR; } - coder = *coder_ptr; + lzma_lzma1_encoder *coder = *coder_ptr; // Set compression mode. We haven't validates the options yet, // but it's OK here, since nothing bad happens with invalid @@ -590,6 +573,7 @@ lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, // Set dist_table_size. // Round the dictionary size up to next 2^n. + uint32_t log_size = 0; while ((UINT32_C(1) << log_size) < options->dict_size) ++log_size; @@ -622,7 +606,7 @@ lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, static lzma_ret -lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, +lzma_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { lz->code = &lzma_encode; @@ -632,7 +616,7 @@ lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { return lzma_lz_encoder_init( @@ -643,19 +627,17 @@ lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern uint64_t lzma_lzma_encoder_memusage(const void *options) { - lzma_lz_options lz_options; - uint64_t lz_memusage; - if (!is_options_valid(options)) return UINT64_MAX; + lzma_lz_options lz_options; set_lz_options(&lz_options, options); - lz_memusage = lzma_lz_encoder_memusage(&lz_options); + const uint64_t lz_memusage = lzma_lz_encoder_memusage(&lz_options); if (lz_memusage == UINT64_MAX) return UINT64_MAX; - return (uint64_t)(sizeof(lzma_coder)) + lz_memusage; + return (uint64_t)(sizeof(lzma_lzma1_encoder)) + lz_memusage; } diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h index abb8d8b..6cfdf22 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h @@ -17,8 +17,12 @@ #include "common.h" +typedef struct lzma_lzma1_encoder_s lzma_lzma1_encoder; + + extern lzma_ret lzma_lzma_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma_encoder_memusage(const void *options); @@ -35,18 +39,18 @@ extern bool lzma_lzma_lclppb_encode( /// Initializes raw LZMA encoder; this is used by LZMA2. extern lzma_ret lzma_lzma_encoder_create( - lzma_coder **coder_ptr, lzma_allocator *allocator, + void **coder_ptr, const lzma_allocator *allocator, const lzma_options_lzma *options, lzma_lz_options *lz_options); /// Resets an already initialized LZMA encoder; this is used by LZMA2. extern lzma_ret lzma_lzma_encoder_reset( - lzma_coder *coder, const lzma_options_lzma *options); + lzma_lzma1_encoder *coder, const lzma_options_lzma *options); -extern lzma_ret lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +extern lzma_ret lzma_lzma_encode(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, uint32_t read_limit); #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c index f983126..6c53d2b 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c @@ -10,6 +10,7 @@ /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder_private.h" +#include "memcmplen.h" #define change_pair(small_dist, big_dist) \ @@ -17,17 +18,10 @@ extern void -lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res) +lzma_lzma_optimum_fast(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res) { - const uint8_t *buf; - uint32_t buf_avail; - uint32_t i; - uint32_t rep_len = 0; - uint32_t rep_index = 0; - uint32_t back_main = 0; - uint32_t limit; - const uint32_t nice_len = mf->nice_len; uint32_t len_main; @@ -40,8 +34,8 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m matches_count = coder->matches_count; } - buf = mf_ptr(mf) - 1; - buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); + const uint8_t *buf = mf_ptr(mf) - 1; + const uint32_t buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); if (buf_avail < 2) { // There's not enough input left to encode a match. @@ -51,9 +45,10 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m } // Look for repeated matches; scan the previous four match distances - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len; + uint32_t rep_len = 0; + uint32_t rep_index = 0; + for (uint32_t i = 0; i < REPS; ++i) { // Pointer to the beginning of the match candidate const uint8_t *const buf_back = buf - coder->reps[i] - 1; @@ -64,8 +59,8 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // The first two bytes matched. // Calculate the length of the match. - for (len = 2; len < buf_avail - && buf[len] == buf_back[len]; ++len) ; + const uint32_t len = lzma_memcmplen( + buf, buf_back, 2, buf_avail); // If we have found a repeated match that is at least // nice_len long, return it immediately. @@ -85,13 +80,13 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // We didn't find a long enough repeated match. Encode it as a normal // match if the match length is at least nice_len. if (len_main >= nice_len) { - *back_res = coder->matches[matches_count - 1].dist - + REP_DISTANCES; + *back_res = coder->matches[matches_count - 1].dist + REPS; *len_res = len_main; mf_skip(mf, len_main - 1); return; } + uint32_t back_main = 0; if (len_main >= 2) { back_main = coder->matches[matches_count - 1].dist; @@ -158,27 +153,17 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // the old buf pointer instead of recalculating it with mf_ptr(). ++buf; - limit = len_main - 1; - - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len; - - const uint8_t *const buf_back = buf - coder->reps[i] - 1; - - if (not_equal_16(buf, buf_back)) - continue; - - for (len = 2; len < limit - && buf[len] == buf_back[len]; ++len) ; + const uint32_t limit = my_max(2, len_main - 1); - if (len >= limit) { + for (uint32_t i = 0; i < REPS; ++i) { + if (memcmp(buf, buf - coder->reps[i] - 1, limit) == 0) { *back_res = UINT32_MAX; *len_res = 1; return; } } - *back_res = back_main + REP_DISTANCES; + *back_res = back_main + REPS; *len_res = len_main; mf_skip(mf, len_main - 2); return; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c index d3a6348..59f7734 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c @@ -11,6 +11,7 @@ #include "lzma_encoder_private.h" #include "fastpos.h" +#include "memcmplen.h" //////////// @@ -18,7 +19,7 @@ //////////// static uint32_t -get_literal_price(const lzma_coder *const coder, const uint32_t pos, +get_literal_price(const lzma_lzma1_encoder *const coder, const uint32_t pos, const uint32_t prev_byte, const bool match_mode, uint32_t match_byte, uint32_t symbol) { @@ -35,15 +36,12 @@ get_literal_price(const lzma_coder *const coder, const uint32_t pos, symbol += UINT32_C(1) << 8; do { - uint32_t match_bit; - uint32_t subcoder_index; - uint32_t bit; - match_byte <<= 1; - match_bit = match_byte & offset; - subcoder_index = offset + match_bit + (symbol >> 8); - bit = (symbol >> 7) & 1; + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; price += rc_bit_price(subcoder[subcoder_index], bit); symbol <<= 1; @@ -67,7 +65,7 @@ get_len_price(const lzma_length_encoder *const lencoder, static inline uint32_t -get_short_rep_price(const lzma_coder *const coder, +get_short_rep_price(const lzma_lzma1_encoder *const coder, const lzma_lzma_state state, const uint32_t pos_state) { return rc_bit_0_price(coder->is_rep0[state]) @@ -76,7 +74,7 @@ get_short_rep_price(const lzma_coder *const coder, static inline uint32_t -get_pure_rep_price(const lzma_coder *const coder, const uint32_t rep_index, +get_pure_rep_price(const lzma_lzma1_encoder *const coder, const uint32_t rep_index, const lzma_lzma_state state, uint32_t pos_state) { uint32_t price; @@ -101,7 +99,7 @@ get_pure_rep_price(const lzma_coder *const coder, const uint32_t rep_index, static inline uint32_t -get_rep_price(const lzma_coder *const coder, const uint32_t rep_index, +get_rep_price(const lzma_lzma1_encoder *const coder, const uint32_t rep_index, const uint32_t len, const lzma_lzma_state state, const uint32_t pos_state) { @@ -111,18 +109,18 @@ get_rep_price(const lzma_coder *const coder, const uint32_t rep_index, static inline uint32_t -get_pos_len_price(const lzma_coder *const coder, const uint32_t pos, +get_dist_len_price(const lzma_lzma1_encoder *const coder, const uint32_t dist, const uint32_t len, const uint32_t pos_state) { - const uint32_t len_to_pos_state = get_len_to_pos_state(len); + const uint32_t dist_state = get_dist_state(len); uint32_t price; - if (pos < FULL_DISTANCES) { - price = coder->distances_prices[len_to_pos_state][pos]; + if (dist < FULL_DISTANCES) { + price = coder->dist_prices[dist_state][dist]; } else { - const uint32_t pos_slot = get_pos_slot_2(pos); - price = coder->pos_slot_prices[len_to_pos_state][pos_slot] - + coder->align_prices[pos & ALIGN_MASK]; + const uint32_t dist_slot = get_dist_slot_2(dist); + price = coder->dist_slot_prices[dist_state][dist_slot] + + coder->align_prices[dist & ALIGN_MASK]; } price += get_len_price(&coder->match_len_encoder, len, pos_state); @@ -132,59 +130,53 @@ get_pos_len_price(const lzma_coder *const coder, const uint32_t pos, static void -fill_distances_prices(lzma_coder *coder) +fill_dist_prices(lzma_lzma1_encoder *coder) { - uint32_t len_to_pos_state; - uint32_t pos_slot; - uint32_t i; - - for (len_to_pos_state = 0; - len_to_pos_state < LEN_TO_POS_STATES; - ++len_to_pos_state) { + for (uint32_t dist_state = 0; dist_state < DIST_STATES; ++dist_state) { - uint32_t *const pos_slot_prices - = coder->pos_slot_prices[len_to_pos_state]; + uint32_t *const dist_slot_prices + = coder->dist_slot_prices[dist_state]; - // Price to encode the pos_slot. - for (pos_slot = 0; - pos_slot < coder->dist_table_size; ++pos_slot) - pos_slot_prices[pos_slot] = rc_bittree_price( - coder->pos_slot[len_to_pos_state], - POS_SLOT_BITS, pos_slot); + // Price to encode the dist_slot. + for (uint32_t dist_slot = 0; + dist_slot < coder->dist_table_size; ++dist_slot) + dist_slot_prices[dist_slot] = rc_bittree_price( + coder->dist_slot[dist_state], + DIST_SLOT_BITS, dist_slot); // For matches with distance >= FULL_DISTANCES, add the price // of the direct bits part of the match distance. (Align bits // are handled by fill_align_prices()). - for (pos_slot = END_POS_MODEL_INDEX; - pos_slot < coder->dist_table_size; ++pos_slot) - pos_slot_prices[pos_slot] += rc_direct_price( - ((pos_slot >> 1) - 1) - ALIGN_BITS); + for (uint32_t dist_slot = DIST_MODEL_END; + dist_slot < coder->dist_table_size; + ++dist_slot) + dist_slot_prices[dist_slot] += rc_direct_price( + ((dist_slot >> 1) - 1) - ALIGN_BITS); // Distances in the range [0, 3] are fully encoded with - // pos_slot, so they are used for coder->distances_prices + // dist_slot, so they are used for coder->dist_prices // as is. - for (i = 0; i < START_POS_MODEL_INDEX; ++i) - coder->distances_prices[len_to_pos_state][i] - = pos_slot_prices[i]; + for (uint32_t i = 0; i < DIST_MODEL_START; ++i) + coder->dist_prices[dist_state][i] + = dist_slot_prices[i]; } - // Distances in the range [4, 127] depend on pos_slot and pos_special. - // We do this in a loop separate from the above loop to avoid - // redundant calls to get_pos_slot(). - for (i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) { - const uint32_t pos_slot = get_pos_slot(i); - const uint32_t footer_bits = ((pos_slot >> 1) - 1); - const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; + // Distances in the range [4, 127] depend on dist_slot and + // dist_special. We do this in a loop separate from the above + // loop to avoid redundant calls to get_dist_slot(). + for (uint32_t i = DIST_MODEL_START; i < FULL_DISTANCES; ++i) { + const uint32_t dist_slot = get_dist_slot(i); + const uint32_t footer_bits = ((dist_slot >> 1) - 1); + const uint32_t base = (2 | (dist_slot & 1)) << footer_bits; const uint32_t price = rc_bittree_reverse_price( - coder->pos_special + base - pos_slot - 1, + coder->dist_special + base - dist_slot - 1, footer_bits, i - base); - for (len_to_pos_state = 0; - len_to_pos_state < LEN_TO_POS_STATES; - ++len_to_pos_state) - coder->distances_prices[len_to_pos_state][i] - = price + coder->pos_slot_prices[ - len_to_pos_state][pos_slot]; + for (uint32_t dist_state = 0; dist_state < DIST_STATES; + ++dist_state) + coder->dist_prices[dist_state][i] + = price + coder->dist_slot_prices[ + dist_state][dist_slot]; } coder->match_price_count = 0; @@ -193,12 +185,11 @@ fill_distances_prices(lzma_coder *coder) static void -fill_align_prices(lzma_coder *coder) +fill_align_prices(lzma_lzma1_encoder *coder) { - uint32_t i; - for (i = 0; i < ALIGN_TABLE_SIZE; ++i) + for (uint32_t i = 0; i < ALIGN_SIZE; ++i) coder->align_prices[i] = rc_bittree_reverse_price( - coder->pos_align, ALIGN_BITS, i); + coder->dist_align, ALIGN_BITS, i); coder->align_price_count = 0; return; @@ -230,18 +221,15 @@ make_short_rep(lzma_optimal *optimal) static void -backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, - uint32_t *LZMA_RESTRICT back_res, uint32_t cur) +backward(lzma_lzma1_encoder *restrict coder, uint32_t *restrict len_res, + uint32_t *restrict back_res, uint32_t cur) { + coder->opts_end_index = cur; + uint32_t pos_mem = coder->opts[cur].pos_prev; uint32_t back_mem = coder->opts[cur].back_prev; - coder->opts_end_index = cur; - do { - const uint32_t pos_prev = pos_mem; - const uint32_t back_cur = back_mem; - if (coder->opts[cur].prev_1_is_literal) { make_literal(&coder->opts[pos_mem]); coder->opts[pos_mem].pos_prev = pos_mem - 1; @@ -256,6 +244,9 @@ backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, } } + const uint32_t pos_prev = pos_mem; + const uint32_t back_cur = back_mem; + back_mem = coder->opts[pos_prev].back_prev; pos_mem = coder->opts[pos_prev].pos_prev; @@ -278,27 +269,10 @@ backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, ////////// static inline uint32_t -helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res, +helper1(lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, uint32_t position) { - uint32_t buf_avail; - const uint8_t *buf; - uint32_t rep_lens[REP_DISTANCES]; - uint32_t rep_max_index = 0; - uint32_t i; - - uint8_t current_byte; - uint8_t match_byte; - - uint32_t pos_state; - uint32_t match_price; - uint32_t rep_match_price; - uint32_t len_end; - uint32_t len; - - uint32_t normal_match_price; - const uint32_t nice_len = mf->nice_len; uint32_t len_main; @@ -312,18 +286,19 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, matches_count = coder->matches_count; } - buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); + const uint32_t buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); if (buf_avail < 2) { *back_res = UINT32_MAX; *len_res = 1; return UINT32_MAX; } - buf = mf_ptr(mf) - 1; + const uint8_t *const buf = mf_ptr(mf) - 1; - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len_test; + uint32_t rep_lens[REPS]; + uint32_t rep_max_index = 0; + for (uint32_t i = 0; i < REPS; ++i) { const uint8_t *const buf_back = buf - coder->reps[i] - 1; if (not_equal_16(buf, buf_back)) { @@ -331,12 +306,9 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, continue; } - for (len_test = 2; len_test < buf_avail - && buf[len_test] == buf_back[len_test]; - ++len_test) ; + rep_lens[i] = lzma_memcmplen(buf, buf_back, 2, buf_avail); - rep_lens[i] = len_test; - if (len_test > rep_lens[rep_max_index]) + if (rep_lens[i] > rep_lens[rep_max_index]) rep_max_index = i; } @@ -349,15 +321,14 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, if (len_main >= nice_len) { - *back_res = coder->matches[matches_count - 1].dist - + REP_DISTANCES; + *back_res = coder->matches[matches_count - 1].dist + REPS; *len_res = len_main; mf_skip(mf, len_main - 1); return UINT32_MAX; } - current_byte = *buf; - match_byte = *(buf - coder->reps[0] - 1); + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - coder->reps[0] - 1); if (len_main < 2 && current_byte != match_byte && rep_lens[rep_max_index] < 2) { @@ -368,7 +339,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, coder->opts[0].state = coder->state; - pos_state = position & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; coder->opts[1].price = rc_bit_0_price( coder->is_match[coder->state][pos_state]) @@ -378,9 +349,9 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, make_literal(&coder->opts[1]); - match_price = rc_bit_1_price( + const uint32_t match_price = rc_bit_1_price( coder->is_match[coder->state][pos_state]); - rep_match_price = match_price + const uint32_t rep_match_price = match_price + rc_bit_1_price(coder->is_rep[coder->state]); if (match_byte == current_byte) { @@ -394,7 +365,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, } } - len_end = my_max(len_main, rep_lens[rep_max_index]); + const uint32_t len_end = my_max(len_main, rep_lens[rep_max_index]); if (len_end < 2) { *back_res = coder->opts[1].back_prev; @@ -404,23 +375,21 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, coder->opts[1].pos_prev = 0; - for (i = 0; i < REP_DISTANCES; ++i) + for (uint32_t i = 0; i < REPS; ++i) coder->opts[0].backs[i] = coder->reps[i]; - len = len_end; + uint32_t len = len_end; do { coder->opts[len].price = RC_INFINITY_PRICE; } while (--len >= 2); - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t price; - + for (uint32_t i = 0; i < REPS; ++i) { uint32_t rep_len = rep_lens[i]; if (rep_len < 2) continue; - price = rep_match_price + get_pure_rep_price( + const uint32_t price = rep_match_price + get_pure_rep_price( coder, i, coder->state, pos_state); do { @@ -439,7 +408,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, } - normal_match_price = match_price + const uint32_t normal_match_price = match_price + rc_bit_0_price(coder->is_rep[coder->state]); len = rep_lens[0] >= 2 ? rep_lens[0] + 1 : 2; @@ -451,14 +420,13 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, for(; ; ++len) { const uint32_t dist = coder->matches[i].dist; const uint32_t cur_and_len_price = normal_match_price - + get_pos_len_price(coder, + + get_dist_len_price(coder, dist, len, pos_state); if (cur_and_len_price < coder->opts[len].price) { coder->opts[len].price = cur_and_len_price; coder->opts[len].pos_prev = 0; - coder->opts[len].back_prev - = dist + REP_DISTANCES; + coder->opts[len].back_prev = dist + REPS; coder->opts[len].prev_1_is_literal = false; } @@ -473,7 +441,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static inline uint32_t -helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, +helper2(lzma_lzma1_encoder *coder, uint32_t *reps, const uint8_t *buf, uint32_t len_end, uint32_t position, const uint32_t cur, const uint32_t nice_len, const uint32_t buf_avail_full) { @@ -481,19 +449,6 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, uint32_t new_len = coder->longest_match_length; uint32_t pos_prev = coder->opts[cur].pos_prev; lzma_lzma_state state; - uint32_t buf_avail; - uint32_t rep_index; - uint32_t i; - - uint32_t cur_price; - uint8_t current_byte; - uint8_t match_byte; - uint32_t pos_state; - uint32_t cur_and_1_price; - bool next_is_literal = false; - uint32_t match_price; - uint32_t rep_match_price; - uint32_t start_len = 2; if (coder->opts[cur].prev_1_is_literal) { --pos_prev; @@ -501,7 +456,7 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (coder->opts[cur].prev_2) { state = coder->opts[coder->opts[cur].pos_prev_2].state; - if (coder->opts[cur].back_prev_2 < REP_DISTANCES) + if (coder->opts[cur].back_prev_2 < REPS) update_long_rep(state); else update_match(state); @@ -530,48 +485,49 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, update_long_rep(state); } else { pos = coder->opts[cur].back_prev; - if (pos < REP_DISTANCES) + if (pos < REPS) update_long_rep(state); else update_match(state); } - if (pos < REP_DISTANCES) { - uint32_t i; - + if (pos < REPS) { reps[0] = coder->opts[pos_prev].backs[pos]; + uint32_t i; for (i = 1; i <= pos; ++i) reps[i] = coder->opts[pos_prev].backs[i - 1]; - for (; i < REP_DISTANCES; ++i) + for (; i < REPS; ++i) reps[i] = coder->opts[pos_prev].backs[i]; } else { - reps[0] = pos - REP_DISTANCES; + reps[0] = pos - REPS; - for (i = 1; i < REP_DISTANCES; ++i) + for (uint32_t i = 1; i < REPS; ++i) reps[i] = coder->opts[pos_prev].backs[i - 1]; } } coder->opts[cur].state = state; - for (i = 0; i < REP_DISTANCES; ++i) + for (uint32_t i = 0; i < REPS; ++i) coder->opts[cur].backs[i] = reps[i]; - cur_price = coder->opts[cur].price; + const uint32_t cur_price = coder->opts[cur].price; - current_byte = *buf; - match_byte = *(buf - reps[0] - 1); + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - reps[0] - 1); - pos_state = position & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; - cur_and_1_price = cur_price + const uint32_t cur_and_1_price = cur_price + rc_bit_0_price(coder->is_match[state][pos_state]) + get_literal_price(coder, position, buf[-1], !is_literal_state(state), match_byte, current_byte); + bool next_is_literal = false; + if (cur_and_1_price < coder->opts[cur + 1].price) { coder->opts[cur + 1].price = cur_and_1_price; coder->opts[cur + 1].pos_prev = cur; @@ -579,9 +535,9 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, next_is_literal = true; } - match_price = cur_price + const uint32_t match_price = cur_price + rc_bit_1_price(coder->is_match[state][pos_state]); - rep_match_price = match_price + const uint32_t rep_match_price = match_price + rc_bit_1_price(coder->is_rep[state]); if (match_byte == current_byte @@ -602,40 +558,31 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (buf_avail_full < 2) return len_end; - buf_avail = my_min(buf_avail_full, nice_len); + const uint32_t buf_avail = my_min(buf_avail_full, nice_len); if (!next_is_literal && match_byte != current_byte) { // speed optimization // try literal + rep0 const uint8_t *const buf_back = buf - reps[0] - 1; const uint32_t limit = my_min(buf_avail_full, nice_len + 1); - uint32_t len_test = 1; - while (len_test < limit && buf[len_test] == buf_back[len_test]) - ++len_test; - - --len_test; + const uint32_t len_test = lzma_memcmplen(buf, buf_back, 1, limit) - 1; if (len_test >= 2) { - uint32_t pos_state_next; - uint32_t next_rep_match_price; - uint32_t offset; - uint32_t cur_and_len_price; - lzma_lzma_state state_2 = state; update_literal(state_2); - pos_state_next = (position + 1) & coder->pos_mask; - next_rep_match_price = cur_and_1_price + const uint32_t pos_state_next = (position + 1) & coder->pos_mask; + const uint32_t next_rep_match_price = cur_and_1_price + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); //for (; len_test >= 2; --len_test) { - offset = cur + 1 + len_test; + const uint32_t offset = cur + 1 + len_test; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; - cur_and_len_price = next_rep_match_price + const uint32_t cur_and_len_price = next_rep_match_price + get_rep_price(coder, 0, len_test, state_2, pos_state_next); @@ -651,23 +598,20 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, } - for (rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) { - uint32_t len_test, len_test_2, len_test_temp; - uint32_t price, limit; + uint32_t start_len = 2; // speed optimization + for (uint32_t rep_index = 0; rep_index < REPS; ++rep_index) { const uint8_t *const buf_back = buf - reps[rep_index] - 1; if (not_equal_16(buf, buf_back)) continue; - for (len_test = 2; len_test < buf_avail - && buf[len_test] == buf_back[len_test]; - ++len_test) ; + uint32_t len_test = lzma_memcmplen(buf, buf_back, 2, buf_avail); while (len_end < cur + len_test) coder->opts[++len_end].price = RC_INFINITY_PRICE; - len_test_temp = len_test; - price = rep_match_price + get_pure_rep_price( + const uint32_t len_test_temp = len_test; + const uint32_t price = rep_match_price + get_pure_rep_price( coder, rep_index, state, pos_state); do { @@ -689,8 +633,8 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, start_len = len_test + 1; - len_test_2 = len_test + 1; - limit = my_min(buf_avail_full, + uint32_t len_test_2 = len_test + 1; + const uint32_t limit = my_min(buf_avail_full, len_test_2 + nice_len); for (; len_test_2 < limit && buf[len_test_2] == buf_back[len_test_2]; @@ -699,18 +643,12 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, len_test_2 -= len_test + 1; if (len_test_2 >= 2) { - uint32_t pos_state_next; - uint32_t cur_and_len_literal_price; - uint32_t next_rep_match_price; - uint32_t offset; - uint32_t cur_and_len_price; - lzma_lzma_state state_2 = state; update_long_rep(state_2); - pos_state_next = (position + len_test) & coder->pos_mask; + uint32_t pos_state_next = (position + len_test) & coder->pos_mask; - cur_and_len_literal_price = price + const uint32_t cur_and_len_literal_price = price + get_len_price(&coder->rep_len_encoder, len_test, pos_state) + rc_bit_0_price(coder->is_match[state_2][pos_state_next]) @@ -722,17 +660,17 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, pos_state_next = (position + len_test + 1) & coder->pos_mask; - next_rep_match_price = cur_and_len_literal_price + const uint32_t next_rep_match_price = cur_and_len_literal_price + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); //for(; len_test_2 >= 2; len_test_2--) { - offset = cur + len_test + 1 + len_test_2; + const uint32_t offset = cur + len_test + 1 + len_test_2; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; - cur_and_len_price = next_rep_match_price + const uint32_t cur_and_len_price = next_rep_match_price + get_rep_price(coder, 0, len_test_2, state_2, pos_state_next); @@ -763,29 +701,27 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (new_len >= start_len) { - uint32_t len_test; - uint32_t i = 0; - const uint32_t normal_match_price = match_price + rc_bit_0_price(coder->is_rep[state]); while (len_end < cur + new_len) coder->opts[++len_end].price = RC_INFINITY_PRICE; + uint32_t i = 0; while (start_len > coder->matches[i].len) ++i; - for (len_test = start_len; ; ++len_test) { + for (uint32_t len_test = start_len; ; ++len_test) { const uint32_t cur_back = coder->matches[i].dist; uint32_t cur_and_len_price = normal_match_price - + get_pos_len_price(coder, + + get_dist_len_price(coder, cur_back, len_test, pos_state); if (cur_and_len_price < coder->opts[cur + len_test].price) { coder->opts[cur + len_test].price = cur_and_len_price; coder->opts[cur + len_test].pos_prev = cur; coder->opts[cur + len_test].back_prev - = cur_back + REP_DISTANCES; + = cur_back + REPS; coder->opts[cur + len_test].prev_1_is_literal = false; } @@ -803,16 +739,12 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, len_test_2 -= len_test + 1; if (len_test_2 >= 2) { - uint32_t pos_state_next; - uint32_t cur_and_len_literal_price; - uint32_t next_rep_match_price; - uint32_t offset; - lzma_lzma_state state_2 = state; update_match(state_2); - pos_state_next = (position + len_test) & coder->pos_mask; + uint32_t pos_state_next + = (position + len_test) & coder->pos_mask; - cur_and_len_literal_price = cur_and_len_price + const uint32_t cur_and_len_literal_price = cur_and_len_price + rc_bit_0_price( coder->is_match[state_2][pos_state_next]) + get_literal_price(coder, @@ -825,14 +757,14 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, update_literal(state_2); pos_state_next = (pos_state_next + 1) & coder->pos_mask; - next_rep_match_price + const uint32_t next_rep_match_price = cur_and_len_literal_price + rc_bit_1_price( coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); // for(; len_test_2 >= 2; --len_test_2) { - offset = cur + len_test + 1 + len_test_2; + const uint32_t offset = cur + len_test + 1 + len_test_2; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; @@ -849,7 +781,7 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, coder->opts[offset].prev_2 = true; coder->opts[offset].pos_prev_2 = cur; coder->opts[offset].back_prev_2 - = cur_back + REP_DISTANCES; + = cur_back + REPS; } //} } @@ -865,14 +797,11 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, extern void -lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res, +lzma_lzma_optimum_normal(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, uint32_t position) { - uint32_t reps[REP_DISTANCES]; - uint32_t len_end; - uint32_t cur; - // If we have symbols pending, return the next pending symbol. if (coder->opts_end_index != coder->opts_current_index) { assert(mf->read_ahead > 0); @@ -889,9 +818,9 @@ lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT // In liblzma they were moved into this single place. if (mf->read_ahead == 0) { if (coder->match_price_count >= (1 << 7)) - fill_distances_prices(coder); + fill_dist_prices(coder); - if (coder->align_price_count >= ALIGN_TABLE_SIZE) + if (coder->align_price_count >= ALIGN_SIZE) fill_align_prices(coder); } @@ -899,13 +828,14 @@ lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT // the original function into two pieces makes it at least a little // more readable, since those two parts don't share many variables. - len_end = helper1(coder, mf, back_res, len_res, position); + uint32_t len_end = helper1(coder, mf, back_res, len_res, position); if (len_end == UINT32_MAX) return; - + uint32_t reps[REPS]; memcpy(reps, coder->reps, sizeof(reps)); + uint32_t cur; for (cur = 1; cur < len_end; ++cur) { assert(cur < OPTS); diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c index 9332abf..711df02 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c @@ -2,6 +2,7 @@ // /// \file lzma_encoder_presets.c /// \brief Encoder presets +/// \note xz needs this even when only decoding is enabled. // // Author: Lasse Collin // @@ -16,10 +17,6 @@ extern LZMA_API(lzma_bool) lzma_lzma_preset(lzma_options_lzma *options, uint32_t preset) { - static const uint8_t dict_pow2[] - = { 18, 20, 21, 22, 22, 23, 23, 24, 25, 26 }; - static const uint8_t depths[] = { 4, 8, 24, 48 }; - const uint32_t level = preset & LZMA_PRESET_LEVEL_MASK; const uint32_t flags = preset & ~LZMA_PRESET_LEVEL_MASK; const uint32_t supported_flags = LZMA_PRESET_EXTREME; @@ -34,12 +31,15 @@ lzma_lzma_preset(lzma_options_lzma *options, uint32_t preset) options->lp = LZMA_LP_DEFAULT; options->pb = LZMA_PB_DEFAULT; + static const uint8_t dict_pow2[] + = { 18, 20, 21, 22, 22, 23, 23, 24, 25, 26 }; options->dict_size = UINT32_C(1) << dict_pow2[level]; if (level <= 3) { options->mode = LZMA_MODE_FAST; options->mf = level == 0 ? LZMA_MF_HC3 : LZMA_MF_HC4; options->nice_len = level <= 1 ? 128 : 273; + static const uint8_t depths[] = { 4, 8, 24, 48 }; options->depth = depths[level]; } else { options->mode = LZMA_MODE_NORMAL; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h index 04fb29e..a2da969 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h @@ -64,12 +64,12 @@ typedef struct { uint32_t pos_prev; // pos_next; uint32_t back_prev; - uint32_t backs[REP_DISTANCES]; + uint32_t backs[REPS]; } lzma_optimal; -struct lzma_coder_s { +struct lzma_lzma1_encoder_s { /// Range encoder lzma_range_encoder rc; @@ -77,7 +77,7 @@ struct lzma_coder_s { lzma_lzma_state state; /// The four most recent match distances - uint32_t reps[REP_DISTANCES]; + uint32_t reps[REPS]; /// Array of match candidates lzma_match matches[MATCH_LEN_MAX + 1]; @@ -112,9 +112,9 @@ struct lzma_coder_s { probability is_rep1[STATES]; probability is_rep2[STATES]; probability is_rep0_long[STATES][POS_STATES_MAX]; - probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; - probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; - probability pos_align[ALIGN_TABLE_SIZE]; + probability dist_slot[DIST_STATES][DIST_SLOTS]; + probability dist_special[FULL_DISTANCES - DIST_MODEL_END]; + probability dist_align[ALIGN_SIZE]; // These are the same as in lzma_decoder.c except that the encoders // include also price tables. @@ -122,12 +122,12 @@ struct lzma_coder_s { lzma_length_encoder rep_len_encoder; // Price tables - uint32_t pos_slot_prices[LEN_TO_POS_STATES][POS_SLOTS]; - uint32_t distances_prices[LEN_TO_POS_STATES][FULL_DISTANCES]; + uint32_t dist_slot_prices[DIST_STATES][DIST_SLOTS]; + uint32_t dist_prices[DIST_STATES][FULL_DISTANCES]; uint32_t dist_table_size; uint32_t match_price_count; - uint32_t align_prices[ALIGN_TABLE_SIZE]; + uint32_t align_prices[ALIGN_SIZE]; uint32_t align_price_count; // Optimal @@ -138,11 +138,11 @@ struct lzma_coder_s { extern void lzma_lzma_optimum_fast( - lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res); + lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res); -extern void lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint32_t *LZMA_RESTRICT back_res, - uint32_t *LZMA_RESTRICT len_res, uint32_t position); +extern void lzma_lzma_optimum_normal(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, uint32_t *restrict back_res, + uint32_t *restrict len_res, uint32_t position); #endif diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_common.h b/Utilities/cmliblzma/liblzma/rangecoder/range_common.h index f15623e..2c74dc1 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_common.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_common.h @@ -40,11 +40,8 @@ // This does the same for a complete bit tree. // (A tree represented as an array.) #define bittree_reset(probs, bit_levels) \ - do { \ - uint32_t bt_i; \ - for (bt_i = 0; bt_i < (1 << (bit_levels)); ++bt_i) \ - bit_reset((probs)[bt_i]); \ - } while (0) + for (uint32_t bt_i = 0; bt_i < (1 << (bit_levels)); ++bt_i) \ + bit_reset((probs)[bt_i]) ////////////////////// diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h b/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h index 199e7b5..e0b051f 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h @@ -25,20 +25,26 @@ typedef struct { /// Reads the first five bytes to initialize the range decoder. -static inline bool -rc_read_init(lzma_range_decoder *rc, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size) +static inline lzma_ret +rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) { while (rc->init_bytes_left > 0) { if (*in_pos == in_size) - return false; + return LZMA_OK; + + // The first byte is always 0x00. It could have been omitted + // in LZMA2 but it wasn't, so one byte is wasted in every + // LZMA2 chunk. + if (rc->init_bytes_left == 5 && in[*in_pos] != 0x00) + return LZMA_DATA_ERROR; rc->code = (rc->code << 8) | in[*in_pos]; ++*in_pos; --rc->init_bytes_left; } - return true; + return LZMA_STREAM_END; } diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h b/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h index e9614f2..1e1c369 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h @@ -115,8 +115,7 @@ rc_direct(lzma_range_encoder *rc, static inline void rc_flush(lzma_range_encoder *rc) { - size_t i; - for (i = 0; i < 5; ++i) + for (size_t i = 0; i < 5; ++i) rc->symbols[rc->count++] = RC_FLUSH; } diff --git a/Utilities/cmliblzma/liblzma/simple/arm.c b/Utilities/cmliblzma/liblzma/simple/arm.c index 8dcba39..181d0e3 100644 --- a/Utilities/cmliblzma/liblzma/simple/arm.c +++ b/Utilities/cmliblzma/liblzma/simple/arm.c @@ -15,19 +15,19 @@ static size_t -arm_code(lzma_simple *simple lzma_attribute((__unused__)), +arm_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { size_t i; for (i = 0; i + 4 <= size; i += 4) { if (buffer[i + 3] == 0xEB) { - uint32_t dest; uint32_t src = (buffer[i + 2] << 16) | (buffer[i + 1] << 8) | (buffer[i + 0]); src <<= 2; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + 8 + src; else @@ -45,7 +45,7 @@ arm_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -arm_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +arm_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -54,7 +54,8 @@ arm_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_arm_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_arm_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return arm_coder_init(next, allocator, filters, true); @@ -62,7 +63,8 @@ lzma_simple_arm_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_arm_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_arm_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return arm_coder_init(next, allocator, filters, false); diff --git a/Utilities/cmliblzma/liblzma/simple/armthumb.c b/Utilities/cmliblzma/liblzma/simple/armthumb.c index 4b890a3..eab4862 100644 --- a/Utilities/cmliblzma/liblzma/simple/armthumb.c +++ b/Utilities/cmliblzma/liblzma/simple/armthumb.c @@ -15,7 +15,7 @@ static size_t -armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), +armthumb_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -23,7 +23,6 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), for (i = 0; i + 4 <= size; i += 2) { if ((buffer[i + 1] & 0xF8) == 0xF0 && (buffer[i + 3] & 0xF8) == 0xF8) { - uint32_t dest; uint32_t src = ((buffer[i + 1] & 0x7) << 19) | (buffer[i + 0] << 11) | ((buffer[i + 3] & 0x7) << 8) @@ -31,6 +30,7 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), src <<= 1; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + 4 + src; else @@ -50,7 +50,7 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -armthumb_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +armthumb_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -60,7 +60,8 @@ armthumb_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_armthumb_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return armthumb_coder_init(next, allocator, filters, true); } @@ -68,7 +69,8 @@ lzma_simple_armthumb_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return armthumb_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/ia64.c b/Utilities/cmliblzma/liblzma/simple/ia64.c index c537cac..580529e 100644 --- a/Utilities/cmliblzma/liblzma/simple/ia64.c +++ b/Utilities/cmliblzma/liblzma/simple/ia64.c @@ -15,7 +15,7 @@ static size_t -ia64_code(lzma_simple *simple lzma_attribute((__unused__)), +ia64_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -28,42 +28,36 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), size_t i; for (i = 0; i + 16 <= size; i += 16) { - size_t slot; - const uint32_t instr_template = buffer[i] & 0x1F; const uint32_t mask = BRANCH_TABLE[instr_template]; uint32_t bit_pos = 5; - for (slot = 0; slot < 3; ++slot, bit_pos += 41) { + for (size_t slot = 0; slot < 3; ++slot, bit_pos += 41) { + if (((mask >> slot) & 1) == 0) + continue; + const size_t byte_pos = (bit_pos >> 3); const uint32_t bit_res = bit_pos & 0x7; uint64_t instruction = 0; - uint64_t inst_norm; - size_t j; - - if (((mask >> slot) & 1) == 0) - continue; - for (j = 0; j < 6; ++j) + for (size_t j = 0; j < 6; ++j) instruction += (uint64_t)( buffer[i + j + byte_pos]) << (8 * j); - inst_norm = instruction >> bit_res; + uint64_t inst_norm = instruction >> bit_res; if (((inst_norm >> 37) & 0xF) == 0x5 && ((inst_norm >> 9) & 0x7) == 0 /* && (inst_norm & 0x3F)== 0 */ ) { - uint32_t dest; - size_t j; - uint32_t src = (uint32_t)( (inst_norm >> 13) & 0xFFFFF); src |= ((inst_norm >> 36) & 1) << 20; src <<= 4; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + src; else @@ -79,7 +73,7 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), instruction &= (1 << bit_res) - 1; instruction |= (inst_norm << bit_res); - for (j = 0; j < 6; j++) + for (size_t j = 0; j < 6; j++) buffer[i + j + byte_pos] = (uint8_t)( instruction >> (8 * j)); @@ -92,7 +86,7 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -ia64_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +ia64_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -102,7 +96,8 @@ ia64_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_ia64_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return ia64_coder_init(next, allocator, filters, true); } @@ -110,7 +105,8 @@ lzma_simple_ia64_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_ia64_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return ia64_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/powerpc.c b/Utilities/cmliblzma/liblzma/simple/powerpc.c index 6f83511..54dfbf1 100644 --- a/Utilities/cmliblzma/liblzma/simple/powerpc.c +++ b/Utilities/cmliblzma/liblzma/simple/powerpc.c @@ -15,7 +15,7 @@ static size_t -powerpc_code(lzma_simple *simple lzma_attribute((__unused__)), +powerpc_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -49,7 +49,7 @@ powerpc_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -powerpc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +powerpc_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -59,7 +59,8 @@ powerpc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_powerpc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return powerpc_coder_init(next, allocator, filters, true); } @@ -67,7 +68,8 @@ lzma_simple_powerpc_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_powerpc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return powerpc_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/simple_coder.c b/Utilities/cmliblzma/liblzma/simple/simple_coder.c index f3bbdd7..13ebabc 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_coder.c +++ b/Utilities/cmliblzma/liblzma/simple/simple_coder.c @@ -18,10 +18,10 @@ /// Copied or encodes/decodes more data to out[]. static lzma_ret -copy_or_code(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +copy_or_code(lzma_simple_coder *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { assert(!coder->end_was_reached); @@ -55,7 +55,7 @@ copy_or_code(lzma_coder *coder, lzma_allocator *allocator, static size_t -call_filter(lzma_coder *coder, uint8_t *buffer, size_t size) +call_filter(lzma_simple_coder *coder, uint8_t *buffer, size_t size) { const size_t filtered = coder->filter(coder->simple, coder->now_pos, coder->is_encoder, @@ -66,13 +66,12 @@ call_filter(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -simple_code(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +simple_code(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { - size_t out_avail; - size_t buf_avail; + lzma_simple_coder *coder = coder_ptr; // TODO: Add partial support for LZMA_SYNC_FLUSH. We can support it // in cases when the filter is able to filter everything. With most @@ -108,13 +107,9 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, // more data to out[] hopefully filling it completely. Then filter // the data in out[]. This step is where most of the data gets // filtered if the buffer sizes used by the application are reasonable. - out_avail = out_size - *out_pos; - buf_avail = coder->size - coder->pos; + const size_t out_avail = out_size - *out_pos; + const size_t buf_avail = coder->size - coder->pos; if (out_avail > buf_avail || buf_avail == 0) { - size_t size; - size_t filtered; - size_t unfiltered; - // Store the old position so that we know from which byte // to start filtering. const size_t out_start = *out_pos; @@ -137,10 +132,11 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, } // Filter out[]. - size = *out_pos - out_start; - filtered = call_filter(coder, out + out_start, size); + const size_t size = *out_pos - out_start; + const size_t filtered = call_filter( + coder, out + out_start, size); - unfiltered = size - filtered; + const size_t unfiltered = size - filtered; assert(unfiltered <= coder->allocated / 2); // Now we can update coder->pos and coder->size, because @@ -204,8 +200,9 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, static void -simple_coder_end(lzma_coder *coder, lzma_allocator *allocator) +simple_coder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_simple_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder->simple, allocator); lzma_free(coder, allocator); @@ -214,10 +211,12 @@ simple_coder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -simple_coder_update(lzma_coder *coder, lzma_allocator *allocator, +simple_coder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_simple_coder *coder = coder_ptr; + // No update support, just call the next filter in the chain. return lzma_next_filter_update( &coder->next, allocator, reversed_filters + 1); @@ -225,59 +224,59 @@ simple_coder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_simple_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size), size_t simple_size, size_t unfiltered_max, uint32_t alignment, bool is_encoder) { - // Allocate memory for the lzma_coder structure if needed. - if (next->coder == NULL) { + // Allocate memory for the lzma_simple_coder structure if needed. + lzma_simple_coder *coder = next->coder; + if (coder == NULL) { // Here we allocate space also for the temporary buffer. We // need twice the size of unfiltered_max, because then it // is always possible to filter at least unfiltered_max bytes // more data in coder->buffer[] if it can be filled completely. - next->coder = lzma_alloc(sizeof(lzma_coder) + coder = lzma_alloc(sizeof(lzma_simple_coder) + 2 * unfiltered_max, allocator); - if (next->coder == NULL) + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &simple_code; next->end = &simple_coder_end; next->update = &simple_coder_update; - next->coder->next = LZMA_NEXT_CODER_INIT; - next->coder->filter = filter; - next->coder->allocated = 2 * unfiltered_max; + coder->next = LZMA_NEXT_CODER_INIT; + coder->filter = filter; + coder->allocated = 2 * unfiltered_max; // Allocate memory for filter-specific data structure. if (simple_size > 0) { - next->coder->simple = lzma_alloc( - simple_size, allocator); - if (next->coder->simple == NULL) + coder->simple = lzma_alloc(simple_size, allocator); + if (coder->simple == NULL) return LZMA_MEM_ERROR; } else { - next->coder->simple = NULL; + coder->simple = NULL; } } if (filters[0].options != NULL) { const lzma_options_bcj *simple = filters[0].options; - next->coder->now_pos = simple->start_offset; - if (next->coder->now_pos & (alignment - 1)) + coder->now_pos = simple->start_offset; + if (coder->now_pos & (alignment - 1)) return LZMA_OPTIONS_ERROR; } else { - next->coder->now_pos = 0; + coder->now_pos = 0; } // Reset variables. - next->coder->is_encoder = is_encoder; - next->coder->end_was_reached = false; - next->coder->pos = 0; - next->coder->filtered = 0; - next->coder->size = 0; - - return lzma_next_filter_init( - &next->coder->next, allocator, filters + 1); + coder->is_encoder = is_encoder; + coder->end_was_reached = false; + coder->pos = 0; + coder->filtered = 0; + coder->size = 0; + + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } diff --git a/Utilities/cmliblzma/liblzma/simple/simple_coder.h b/Utilities/cmliblzma/liblzma/simple/simple_coder.h index 0952fad..19c2ee0 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_coder.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_coder.h @@ -17,44 +17,56 @@ extern lzma_ret lzma_simple_x86_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_x86_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_powerpc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_powerpc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_ia64_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_ia64_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_arm_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_arm_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_armthumb_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/simple/simple_decoder.c b/Utilities/cmliblzma/liblzma/simple/simple_decoder.c index 034e158..1d864f2 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_decoder.c +++ b/Utilities/cmliblzma/liblzma/simple/simple_decoder.c @@ -14,18 +14,17 @@ extern lzma_ret -lzma_simple_props_decode(void **options, lzma_allocator *allocator, +lzma_simple_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_bcj *opt; - if (props_size == 0) return LZMA_OK; if (props_size != 4) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_bcj), allocator); + lzma_options_bcj *opt = lzma_alloc( + sizeof(lzma_options_bcj), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/simple/simple_decoder.h b/Utilities/cmliblzma/liblzma/simple/simple_decoder.h index b8bf590..bed8d37 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_decoder.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_decoder.h @@ -16,7 +16,7 @@ #include "simple_coder.h" extern lzma_ret lzma_simple_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/simple/simple_private.h b/Utilities/cmliblzma/liblzma/simple/simple_private.h index fcf9f7c..9d2c0fd 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_private.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_private.h @@ -16,9 +16,7 @@ #include "simple_coder.h" -typedef struct lzma_simple_s lzma_simple; - -struct lzma_coder_s { +typedef struct { /// Next filter in the chain lzma_next_coder next; @@ -33,12 +31,12 @@ struct lzma_coder_s { /// Pointer to filter-specific function, which does /// the actual filtering. - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size); /// Pointer to filter-specific data, or NULL if filter doesn't need /// any extra data. - lzma_simple *simple; + void *simple; /// The lowest 32 bits of the current position in the data. Most /// filters need this to do conversions between absolute and relative @@ -62,12 +60,13 @@ struct lzma_coder_s { /// Temporary buffer uint8_t buffer[]; -}; +} lzma_simple_coder; extern lzma_ret lzma_simple_coder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters, - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + const lzma_allocator *allocator, + const lzma_filter_info *filters, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size), size_t simple_size, size_t unfiltered_max, uint32_t alignment, bool is_encoder); diff --git a/Utilities/cmliblzma/liblzma/simple/sparc.c b/Utilities/cmliblzma/liblzma/simple/sparc.c index 0ddd2ac..74b2655 100644 --- a/Utilities/cmliblzma/liblzma/simple/sparc.c +++ b/Utilities/cmliblzma/liblzma/simple/sparc.c @@ -15,7 +15,7 @@ static size_t -sparc_code(lzma_simple *simple lzma_attribute((__unused__)), +sparc_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -26,8 +26,6 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), || (buffer[i] == 0x7F && (buffer[i + 1] & 0xC0) == 0xC0)) { - uint32_t dest; - uint32_t src = ((uint32_t)buffer[i + 0] << 24) | ((uint32_t)buffer[i + 1] << 16) | ((uint32_t)buffer[i + 2] << 8) @@ -35,6 +33,7 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), src <<= 2; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + src; else @@ -58,7 +57,7 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -sparc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +sparc_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -68,7 +67,8 @@ sparc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return sparc_coder_init(next, allocator, filters, true); } @@ -76,7 +76,8 @@ lzma_simple_sparc_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return sparc_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/x86.c b/Utilities/cmliblzma/liblzma/simple/x86.c index 95858e5..23d0c42 100644 --- a/Utilities/cmliblzma/liblzma/simple/x86.c +++ b/Utilities/cmliblzma/liblzma/simple/x86.c @@ -17,14 +17,14 @@ #define Test86MSByte(b) ((b) == 0 || (b) == 0xFF) -struct lzma_simple_s { +typedef struct { uint32_t prev_mask; uint32_t prev_pos; -}; +} lzma_simple_x86; static size_t -x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, +x86_code(void *simple_ptr, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { static const bool MASK_TO_ALLOWED_STATUS[8] @@ -33,39 +33,34 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, static const uint32_t MASK_TO_BIT_NUMBER[8] = { 0, 1, 2, 2, 3, 3, 3, 3 }; + lzma_simple_x86 *simple = simple_ptr; uint32_t prev_mask = simple->prev_mask; uint32_t prev_pos = simple->prev_pos; - size_t limit; - size_t buffer_pos; - if (size < 5) return 0; if (now_pos - prev_pos > 5) prev_pos = now_pos - 5; - limit = size - 5; - buffer_pos = 0; + const size_t limit = size - 5; + size_t buffer_pos = 0; while (buffer_pos <= limit) { - uint32_t offset; - uint32_t i; - uint8_t b = buffer[buffer_pos]; if (b != 0xE8 && b != 0xE9) { ++buffer_pos; continue; } - offset = now_pos + (uint32_t)(buffer_pos) + const uint32_t offset = now_pos + (uint32_t)(buffer_pos) - prev_pos; prev_pos = now_pos + (uint32_t)(buffer_pos); if (offset > 5) { prev_mask = 0; } else { - for (i = 0; i < offset; ++i) { + for (uint32_t i = 0; i < offset; ++i) { prev_mask &= 0x77; prev_mask <<= 1; } @@ -84,8 +79,6 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, uint32_t dest; while (true) { - uint32_t i; - if (is_encoder) dest = src + (now_pos + (uint32_t)( buffer_pos) + 5); @@ -96,7 +89,8 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, if (prev_mask == 0) break; - i = MASK_TO_BIT_NUMBER[prev_mask >> 1]; + const uint32_t i = MASK_TO_BIT_NUMBER[ + prev_mask >> 1]; b = (uint8_t)(dest >> (24 - i * 8)); @@ -130,15 +124,17 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, static lzma_ret -x86_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +x86_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { const lzma_ret ret = lzma_simple_coder_init(next, allocator, filters, - &x86_code, sizeof(lzma_simple), 5, 1, is_encoder); + &x86_code, sizeof(lzma_simple_x86), 5, 1, is_encoder); if (ret == LZMA_OK) { - next->coder->simple->prev_mask = 0; - next->coder->simple->prev_pos = (uint32_t)(-5); + lzma_simple_coder *coder = next->coder; + lzma_simple_x86 *simple = coder->simple; + simple->prev_mask = 0; + simple->prev_pos = (uint32_t)(-5); } return ret; @@ -146,7 +142,8 @@ x86_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_x86_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_x86_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return x86_coder_init(next, allocator, filters, true); @@ -154,7 +151,8 @@ lzma_simple_x86_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_x86_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_x86_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return x86_coder_init(next, allocator, filters, false); diff --git a/Utilities/cmlibrhash/librhash/rhash.c b/Utilities/cmlibrhash/librhash/rhash.c index ad6249b..34e1eb3 100644 --- a/Utilities/cmlibrhash/librhash/rhash.c +++ b/Utilities/cmlibrhash/librhash/rhash.c @@ -18,6 +18,8 @@ #define _LARGEFILE64_SOURCE #define _FILE_OFFSET_BITS 64 +#include "ustd.h" /* Need this first within CMake. */ + #include <string.h> /* memset() */ #include <stdlib.h> /* free() */ #include <stddef.h> /* ptrdiff_t */ |