diff options
Diffstat (limited to 'Utilities')
291 files changed, 7819 insertions, 4256 deletions
diff --git a/Utilities/IWYU/mapping.imp b/Utilities/IWYU/mapping.imp index 892cdcb..5af8b85 100644 --- a/Utilities/IWYU/mapping.imp +++ b/Utilities/IWYU/mapping.imp @@ -52,6 +52,9 @@ #{ symbol: [ "std::pair", private, "<map>", public ] }, #{ symbol: [ "std::pair", private, "<set>", public ] }, + # HACK: iwyu wrongly thinks that <system_error> is needed for std::hash + { symbol: [ "std::hash", private, "<functional>", public ] }, + # __decay_and_strip is used internally in the C++11 standard library. # IWYU does not classify it as internal and suggests to add <type_traits>. # To ignore it, we simply map it to a file that is included anyway. diff --git a/Utilities/Release/CMakeLogo.ico b/Utilities/Release/CMakeLogo.ico Binary files differindex e13bb15..c100612 100644 --- a/Utilities/Release/CMakeLogo.ico +++ b/Utilities/Release/CMakeLogo.ico diff --git a/Utilities/Release/create-cmake-release.cmake b/Utilities/Release/create-cmake-release.cmake index 3af1b03..b3cc352 100644 --- a/Utilities/Release/create-cmake-release.cmake +++ b/Utilities/Release/create-cmake-release.cmake @@ -54,7 +54,7 @@ cd \${name}-build && -DCMAKE_INSTALL_PREFIX=\"\$inst/\" \\ -DCMAKE_DOC_DIR=doc/cmake \\ -DSPHINX_EXECUTABLE=\"${SPHINX_EXECUTABLE}\" \\ - -DSPHINX_HTML=ON -DSPHINX_MAN=ON && + -DSPHINX_HTML=ON -DSPHINX_MAN=ON -DSPHINX_QTHELP=ON && make install && cd .. && tar czf \${name}.tar.gz \${name} || diff --git a/Utilities/Release/linux64_release.cmake b/Utilities/Release/linux64_release.cmake index 04a74ac..dc34120 100644 --- a/Utilities/Release/linux64_release.cmake +++ b/Utilities/Release/linux64_release.cmake @@ -3,8 +3,8 @@ set(BOOTSTRAP_ARGS "--docdir=doc/cmake") set(HOST linux64) set(MAKE_PROGRAM "make") set(CPACK_BINARY_GENERATORS "STGZ TGZ") -set(CC /opt/gcc-6.1.0/bin/gcc) -set(CXX /opt/gcc-6.1.0/bin/g++) +set(CC /opt/gcc-8.2.0/bin/gcc) +set(CXX /opt/gcc-8.2.0/bin/g++) set(CFLAGS "") set(CXXFLAGS "") set(qt_prefix "/home/kitware/qt-5.7.0") @@ -34,12 +34,16 @@ OPENSSL_INCLUDE_DIR:PATH=/home/kitware/openssl-1.1.0h/include OPENSSL_SSL_LIBRARY:FILEPATH=/home/kitware/openssl-1.1.0h/lib/libssl.a PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3 CPACK_SYSTEM_NAME:STRING=Linux-x86_64 +BUILD_CursesDialog:BOOL=ON BUILD_QtDialog:BOOL=TRUE CMAKE_SKIP_BOOTSTRAP_TEST:STRING=TRUE CMake_GUI_DISTRIBUTE_WITH_Qt_LGPL:STRING=3 CMAKE_PREFIX_PATH:STRING=${qt_prefix} CMake_QT_STATIC_QXcbIntegrationPlugin_LIBRARIES:STRING=${qt_xcb_libs} ") +set(ENV [[ +export CMAKE_PREFIX_PATH=/opt/binutils-2.31 +]]) # Exclude Qt5 tests because our Qt5 is static. set(EXTRA_CTEST_ARGS "-E Qt5") diff --git a/Utilities/Release/osx_release.cmake b/Utilities/Release/osx_release.cmake index c69eb11..be11d47 100644 --- a/Utilities/Release/osx_release.cmake +++ b/Utilities/Release/osx_release.cmake @@ -19,6 +19,7 @@ CMAKE_OSX_ARCHITECTURES:STRING=x86_64 CMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7 CMAKE_SKIP_BOOTSTRAP_TEST:STRING=TRUE CPACK_SYSTEM_NAME:STRING=Darwin-x86_64 +BUILD_CursesDialog:BOOL=ON BUILD_QtDialog:BOOL=TRUE CMake_GUI_DISTRIBUTE_WITH_Qt_LGPL:STRING=3 CMake_INSTALL_DEPENDENCIES:BOOL=ON diff --git a/Utilities/Release/upload_release.cmake b/Utilities/Release/upload_release.cmake index 1edcd65..bbc7437 100644 --- a/Utilities/Release/upload_release.cmake +++ b/Utilities/Release/upload_release.cmake @@ -1,6 +1,6 @@ set(CTEST_RUN_CURRENT_SCRIPT 0) if(NOT VERSION) - set(VERSION 3.12) + set(VERSION 3.13) endif() if(NOT DEFINED PROJECT_PREFIX) set(PROJECT_PREFIX cmake-${VERSION}) diff --git a/Utilities/Release/win32_release.cmake b/Utilities/Release/win32_release.cmake index f9e35a5..2e817d9 100644 --- a/Utilities/Release/win32_release.cmake +++ b/Utilities/Release/win32_release.cmake @@ -39,6 +39,6 @@ get_filename_component(path "${CMAKE_CURRENT_LIST_FILE}" PATH) set(GIT_EXTRA "git config core.autocrlf true") if(CMAKE_CREATE_VERSION STREQUAL "nightly") # Some tests fail spuriously too often. - set(EXTRA_CTEST_ARGS "-E 'ConsoleBuf'") + set(EXTRA_CTEST_ARGS "-E 'ConsoleBuf|Module.ExternalData'") endif() include(${path}/release_cmake.cmake) diff --git a/Utilities/Release/win64_release.cmake b/Utilities/Release/win64_release.cmake index 02e4096..33af830 100644 --- a/Utilities/Release/win64_release.cmake +++ b/Utilities/Release/win64_release.cmake @@ -39,6 +39,6 @@ get_filename_component(path "${CMAKE_CURRENT_LIST_FILE}" PATH) set(GIT_EXTRA "git config core.autocrlf true") if(CMAKE_CREATE_VERSION STREQUAL "nightly") # Some tests fail spuriously too often. - set(EXTRA_CTEST_ARGS "-E 'ConsoleBuf'") + set(EXTRA_CTEST_ARGS "-E 'ConsoleBuf|Module.ExternalData'") endif() include(${path}/release_cmake.cmake) diff --git a/Utilities/Scripts/update-curl.bash b/Utilities/Scripts/update-curl.bash index b02180f..3d31e38 100755 --- a/Utilities/Scripts/update-curl.bash +++ b/Utilities/Scripts/update-curl.bash @@ -8,7 +8,7 @@ readonly name="curl" readonly ownership="Curl Upstream <curl-library@cool.haxx.se>" readonly subtree="Utilities/cmcurl" readonly repo="https://github.com/curl/curl.git" -readonly tag="curl-7_60_0" +readonly tag="curl-7_61_1" readonly shortlog=false readonly paths=" CMake/* diff --git a/Utilities/Scripts/update-libarchive.bash b/Utilities/Scripts/update-libarchive.bash index 7534f94..3188658 100755 --- a/Utilities/Scripts/update-libarchive.bash +++ b/Utilities/Scripts/update-libarchive.bash @@ -8,7 +8,7 @@ readonly name="LibArchive" readonly ownership="LibArchive Upstream <libarchive-discuss@googlegroups.com>" readonly subtree="Utilities/cmlibarchive" readonly repo="https://github.com/libarchive/libarchive.git" -readonly tag="v3.3.2" +readonly tag="v3.3.3" readonly shortlog=false readonly paths=" CMakeLists.txt diff --git a/Utilities/Scripts/update-liblzma.bash b/Utilities/Scripts/update-liblzma.bash index 088eb91..fdf66b3 100755 --- a/Utilities/Scripts/update-liblzma.bash +++ b/Utilities/Scripts/update-liblzma.bash @@ -7,8 +7,8 @@ shopt -s dotglob readonly name="liblzma" readonly ownership="liblzma upstream <xz-devel@tukaani.org>" readonly subtree="Utilities/cmliblzma" -readonly repo="http://git.tukaani.org/xz.git" -readonly tag="v5.0.8" +readonly repo="https://git.tukaani.org/xz.git" +readonly tag="v5.2.4" readonly shortlog=false readonly paths=" COPYING @@ -24,6 +24,10 @@ extract_source () { mv src/common . mv src/liblzma . rmdir src + rm liblzma/Makefile.* + rm liblzma/*/Makefile.* + rm liblzma/liblzma.map + rm liblzma/validate_map.sh popd } diff --git a/Utilities/Sphinx/cmake.py b/Utilities/Sphinx/cmake.py index b0c8f71..ebf44da 100644 --- a/Utilities/Sphinx/cmake.py +++ b/Utilities/Sphinx/cmake.py @@ -144,6 +144,7 @@ class _cmake_index_entry: _cmake_index_objs = { 'command': _cmake_index_entry('command'), + 'cpack_gen': _cmake_index_entry('cpack generator'), 'envvar': _cmake_index_entry('envvar'), 'generator': _cmake_index_entry('generator'), 'manual': _cmake_index_entry('manual'), @@ -325,6 +326,7 @@ class CMakeDomain(Domain): label = 'CMake' object_types = { 'command': ObjType('command', 'command'), + 'cpack_gen': ObjType('cpack_gen', 'cpack_gen'), 'envvar': ObjType('envvar', 'envvar'), 'generator': ObjType('generator', 'generator'), 'variable': ObjType('variable', 'variable'), @@ -358,6 +360,7 @@ class CMakeDomain(Domain): } roles = { 'command': CMakeXRefRole(fix_parens = True, lowercase = True), + 'cpack_gen': CMakeXRefRole(), 'envvar': CMakeXRefRole(), 'generator': CMakeXRefRole(), 'variable': CMakeXRefRole(), diff --git a/Utilities/Sphinx/create_identifiers.py b/Utilities/Sphinx/create_identifiers.py index e638950..6716b48 100755 --- a/Utilities/Sphinx/create_identifiers.py +++ b/Utilities/Sphinx/create_identifiers.py @@ -21,6 +21,7 @@ newlines = [] for line in lines: mapping = (("command", "command"), + ("cpack generator", "cpack_gen"), ("envvar", "envvar"), ("variable", "variable"), ("generator", "generator"), diff --git a/Utilities/Sphinx/static/cmake-favicon.ico b/Utilities/Sphinx/static/cmake-favicon.ico Binary files differindex fce8f92..ae529f5 100644 --- a/Utilities/Sphinx/static/cmake-favicon.ico +++ b/Utilities/Sphinx/static/cmake-favicon.ico diff --git a/Utilities/Sphinx/static/cmake-logo-16.png b/Utilities/Sphinx/static/cmake-logo-16.png Binary files differindex 2039c25..9c95b75 100644 --- a/Utilities/Sphinx/static/cmake-logo-16.png +++ b/Utilities/Sphinx/static/cmake-logo-16.png diff --git a/Utilities/cmcurl/CMake/CurlSymbolHiding.cmake b/Utilities/cmcurl/CMake/CurlSymbolHiding.cmake index 9f7d296..15ba46e 100644 --- a/Utilities/cmcurl/CMake/CurlSymbolHiding.cmake +++ b/Utilities/cmcurl/CMake/CurlSymbolHiding.cmake @@ -4,57 +4,57 @@ option(CURL_HIDDEN_SYMBOLS "Set to ON to hide libcurl internal symbols (=hide al mark_as_advanced(CURL_HIDDEN_SYMBOLS) if(CURL_HIDDEN_SYMBOLS) - set(SUPPORTS_SYMBOL_HIDING FALSE) + set(SUPPORTS_SYMBOL_HIDING FALSE) - if(CMAKE_C_COMPILER_ID MATCHES "Clang") - set(SUPPORTS_SYMBOL_HIDING TRUE) - set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") - set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") - elseif(CMAKE_COMPILER_IS_GNUCC) - if(NOT CMAKE_VERSION VERSION_LESS 2.8.10) - set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION}) - else() - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion - OUTPUT_VARIABLE GCC_VERSION) - endif() - if(NOT GCC_VERSION VERSION_LESS 3.4) - # note: this is considered buggy prior to 4.0 but the autotools don't care, so let's ignore that fact - set(SUPPORTS_SYMBOL_HIDING TRUE) - set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") - set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") - endif() - elseif(CMAKE_C_COMPILER_ID MATCHES "SunPro" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0) - set(SUPPORTS_SYMBOL_HIDING TRUE) - set(_SYMBOL_EXTERN "__global") - set(_CFLAG_SYMBOLS_HIDE "-xldscope=hidden") - elseif(CMAKE_C_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0) - # note: this should probably just check for version 9.1.045 but I'm not 100% sure - # so let's to it the same way autotools do. - set(SUPPORTS_SYMBOL_HIDING TRUE) - set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") - set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") - check_c_source_compiles("#include <stdio.h> - int main (void) { printf(\"icc fvisibility bug test\"); return 0; }" _no_bug) - if(NOT _no_bug) - set(SUPPORTS_SYMBOL_HIDING FALSE) - set(_SYMBOL_EXTERN "") - set(_CFLAG_SYMBOLS_HIDE "") - endif() - elseif(MSVC) - set(SUPPORTS_SYMBOL_HIDING TRUE) + if(CMAKE_C_COMPILER_ID MATCHES "Clang") + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") + set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") + elseif(CMAKE_COMPILER_IS_GNUCC) + if(NOT CMAKE_VERSION VERSION_LESS 2.8.10) + set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION}) + else() + execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion + OUTPUT_VARIABLE GCC_VERSION) + endif() + if(NOT GCC_VERSION VERSION_LESS 3.4) + # note: this is considered buggy prior to 4.0 but the autotools don't care, so let's ignore that fact + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") + set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") endif() + elseif(CMAKE_C_COMPILER_ID MATCHES "SunPro" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0) + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__global") + set(_CFLAG_SYMBOLS_HIDE "-xldscope=hidden") + elseif(CMAKE_C_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0) + # note: this should probably just check for version 9.1.045 but I'm not 100% sure + # so let's to it the same way autotools do. + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") + set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") + check_c_source_compiles("#include <stdio.h> + int main (void) { printf(\"icc fvisibility bug test\"); return 0; }" _no_bug) + if(NOT _no_bug) + set(SUPPORTS_SYMBOL_HIDING FALSE) + set(_SYMBOL_EXTERN "") + set(_CFLAG_SYMBOLS_HIDE "") + endif() + elseif(MSVC) + set(SUPPORTS_SYMBOL_HIDING TRUE) + endif() - set(HIDES_CURL_PRIVATE_SYMBOLS ${SUPPORTS_SYMBOL_HIDING}) + set(HIDES_CURL_PRIVATE_SYMBOLS ${SUPPORTS_SYMBOL_HIDING}) elseif(MSVC) - if(NOT CMAKE_VERSION VERSION_LESS 3.7) - set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) #present since 3.4.3 but broken - set(HIDES_CURL_PRIVATE_SYMBOLS FALSE) - else() - message(WARNING "Hiding private symbols regardless CURL_HIDDEN_SYMBOLS being disabled.") - set(HIDES_CURL_PRIVATE_SYMBOLS TRUE) - endif() -elseif() + if(NOT CMAKE_VERSION VERSION_LESS 3.7) + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) #present since 3.4.3 but broken set(HIDES_CURL_PRIVATE_SYMBOLS FALSE) + else() + message(WARNING "Hiding private symbols regardless CURL_HIDDEN_SYMBOLS being disabled.") + set(HIDES_CURL_PRIVATE_SYMBOLS TRUE) + endif() +elseif() + set(HIDES_CURL_PRIVATE_SYMBOLS FALSE) endif() set(CURL_CFLAG_SYMBOLS_HIDE ${_CFLAG_SYMBOLS_HIDE}) diff --git a/Utilities/cmcurl/CMake/CurlTests.c b/Utilities/cmcurl/CMake/CurlTests.c index bc36c8e..ab244ac 100644 --- a/Utilities/cmcurl/CMake/CurlTests.c +++ b/Utilities/cmcurl/CMake/CurlTests.c @@ -507,30 +507,30 @@ main () #ifdef HAVE_GLIBC_STRERROR_R #include <string.h> #include <errno.h> + +void check(char c) {} + int main () { - char buffer[1024]; /* big enough to play with */ - char *string = - strerror_r(EACCES, buffer, sizeof(buffer)); - /* this should've returned a string */ - if(!string || !string[0]) - return 99; - return 0; + char buffer[1024]; + /* This will not compile if strerror_r does not return a char* */ + check(strerror_r(EACCES, buffer, sizeof(buffer))[0]); + return 0; } #endif #ifdef HAVE_POSIX_STRERROR_R #include <string.h> #include <errno.h> + +/* float, because a pointer can't be implicitly cast to float */ +void check(float f) {} + int main () { - char buffer[1024]; /* big enough to play with */ - int error = - strerror_r(EACCES, buffer, sizeof(buffer)); - /* This should've returned zero, and written an error string in the - buffer.*/ - if(!buffer[0] || error) - return 99; - return 0; + char buffer[1024]; + /* This will not compile if strerror_r does not return an int */ + check(strerror_r(EACCES, buffer, sizeof(buffer))); + return 0; } #endif #ifdef HAVE_FSETXATTR_6 diff --git a/Utilities/cmcurl/CMake/FindCARES.cmake b/Utilities/cmcurl/CMake/FindCARES.cmake index c4ab5f1..723044a 100644 --- a/Utilities/cmcurl/CMake/FindCARES.cmake +++ b/Utilities/cmcurl/CMake/FindCARES.cmake @@ -7,36 +7,36 @@ # also defined, but not for general use are # CARES_LIBRARY, where to find the c-ares library. -FIND_PATH(CARES_INCLUDE_DIR ares.h +find_path(CARES_INCLUDE_DIR ares.h /usr/local/include /usr/include ) -SET(CARES_NAMES ${CARES_NAMES} cares) -FIND_LIBRARY(CARES_LIBRARY +set(CARES_NAMES ${CARES_NAMES} cares) +find_library(CARES_LIBRARY NAMES ${CARES_NAMES} PATHS /usr/lib /usr/local/lib ) -IF (CARES_LIBRARY AND CARES_INCLUDE_DIR) - SET(CARES_LIBRARIES ${CARES_LIBRARY}) - SET(CARES_FOUND "YES") -ELSE (CARES_LIBRARY AND CARES_INCLUDE_DIR) - SET(CARES_FOUND "NO") -ENDIF (CARES_LIBRARY AND CARES_INCLUDE_DIR) +if(CARES_LIBRARY AND CARES_INCLUDE_DIR) + set(CARES_LIBRARIES ${CARES_LIBRARY}) + set(CARES_FOUND "YES") +else() + set(CARES_FOUND "NO") +endif() -IF (CARES_FOUND) - IF (NOT CARES_FIND_QUIETLY) - MESSAGE(STATUS "Found c-ares: ${CARES_LIBRARIES}") - ENDIF (NOT CARES_FIND_QUIETLY) -ELSE (CARES_FOUND) - IF (CARES_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find c-ares library") - ENDIF (CARES_FIND_REQUIRED) -ENDIF (CARES_FOUND) +if(CARES_FOUND) + if(NOT CARES_FIND_QUIETLY) + message(STATUS "Found c-ares: ${CARES_LIBRARIES}") + endif() +else() + if(CARES_FIND_REQUIRED) + message(FATAL_ERROR "Could not find c-ares library") + endif() +endif() -MARK_AS_ADVANCED( +mark_as_advanced( CARES_LIBRARY CARES_INCLUDE_DIR ) diff --git a/Utilities/cmcurl/CMake/FindGSS.cmake b/Utilities/cmcurl/CMake/FindGSS.cmake index 60dcb73..7a637fc 100644 --- a/Utilities/cmcurl/CMake/FindGSS.cmake +++ b/Utilities/cmcurl/CMake/FindGSS.cmake @@ -28,211 +28,209 @@ set(_GSS_ROOT_HINTS # try to find library using system pkg-config if user didn't specify root dir if(NOT GSS_ROOT_DIR AND NOT "$ENV{GSS_ROOT_DIR}") - if(UNIX) - find_package(PkgConfig QUIET) - pkg_search_module(_GSS_PKG ${_MIT_MODNAME} ${_HEIMDAL_MODNAME}) - list(APPEND _GSS_ROOT_HINTS "${_GSS_PKG_PREFIX}") - elseif(WIN32) - list(APPEND _GSS_ROOT_HINTS "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MIT\\Kerberos;InstallDir]") - endif() + if(UNIX) + find_package(PkgConfig QUIET) + pkg_search_module(_GSS_PKG ${_MIT_MODNAME} ${_HEIMDAL_MODNAME}) + list(APPEND _GSS_ROOT_HINTS "${_GSS_PKG_PREFIX}") + elseif(WIN32) + list(APPEND _GSS_ROOT_HINTS "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MIT\\Kerberos;InstallDir]") + endif() endif() if(NOT _GSS_FOUND) #not found by pkg-config. Let's take more traditional approach. - find_file(_GSS_CONFIGURE_SCRIPT - NAMES - "krb5-config" - HINTS - ${_GSS_ROOT_HINTS} - PATH_SUFFIXES - bin - NO_CMAKE_PATH - NO_CMAKE_ENVIRONMENT_PATH + find_file(_GSS_CONFIGURE_SCRIPT + NAMES + "krb5-config" + HINTS + ${_GSS_ROOT_HINTS} + PATH_SUFFIXES + bin + NO_CMAKE_PATH + NO_CMAKE_ENVIRONMENT_PATH + ) + + # if not found in user-supplied directories, maybe system knows better + find_file(_GSS_CONFIGURE_SCRIPT + NAMES + "krb5-config" + PATH_SUFFIXES + bin + ) + + if(_GSS_CONFIGURE_SCRIPT) + execute_process( + COMMAND ${_GSS_CONFIGURE_SCRIPT} "--cflags" "gssapi" + OUTPUT_VARIABLE _GSS_CFLAGS + RESULT_VARIABLE _GSS_CONFIGURE_FAILED + ) + message(STATUS "CFLAGS: ${_GSS_CFLAGS}") + if(NOT _GSS_CONFIGURE_FAILED) # 0 means success + # should also work in an odd case when multiple directories are given + string(STRIP "${_GSS_CFLAGS}" _GSS_CFLAGS) + string(REGEX REPLACE " +-I" ";" _GSS_CFLAGS "${_GSS_CFLAGS}") + string(REGEX REPLACE " +-([^I][^ \\t;]*)" ";-\\1"_GSS_CFLAGS "${_GSS_CFLAGS}") + + foreach(_flag ${_GSS_CFLAGS}) + if(_flag MATCHES "^-I.*") + string(REGEX REPLACE "^-I" "" _val "${_flag}") + list(APPEND _GSS_INCLUDE_DIR "${_val}") + else() + list(APPEND _GSS_COMPILER_FLAGS "${_flag}") + endif() + endforeach() + endif() + + execute_process( + COMMAND ${_GSS_CONFIGURE_SCRIPT} "--libs" "gssapi" + OUTPUT_VARIABLE _GSS_LIB_FLAGS + RESULT_VARIABLE _GSS_CONFIGURE_FAILED ) + message(STATUS "LDFLAGS: ${_GSS_LIB_FLAGS}") + + if(NOT _GSS_CONFIGURE_FAILED) # 0 means success + # this script gives us libraries and link directories. Blah. We have to deal with it. + string(STRIP "${_GSS_LIB_FLAGS}" _GSS_LIB_FLAGS) + string(REGEX REPLACE " +-(L|l)" ";-\\1" _GSS_LIB_FLAGS "${_GSS_LIB_FLAGS}") + string(REGEX REPLACE " +-([^Ll][^ \\t;]*)" ";-\\1"_GSS_LIB_FLAGS "${_GSS_LIB_FLAGS}") + + foreach(_flag ${_GSS_LIB_FLAGS}) + if(_flag MATCHES "^-l.*") + string(REGEX REPLACE "^-l" "" _val "${_flag}") + list(APPEND _GSS_LIBRARIES "${_val}") + elseif(_flag MATCHES "^-L.*") + string(REGEX REPLACE "^-L" "" _val "${_flag}") + list(APPEND _GSS_LINK_DIRECTORIES "${_val}") + else() + list(APPEND _GSS_LINKER_FLAGS "${_flag}") + endif() + endforeach() + endif() - # if not found in user-supplied directories, maybe system knows better - find_file(_GSS_CONFIGURE_SCRIPT - NAMES - "krb5-config" - PATH_SUFFIXES - bin + execute_process( + COMMAND ${_GSS_CONFIGURE_SCRIPT} "--version" + OUTPUT_VARIABLE _GSS_VERSION + RESULT_VARIABLE _GSS_CONFIGURE_FAILED ) - if(_GSS_CONFIGURE_SCRIPT) - execute_process( - COMMAND ${_GSS_CONFIGURE_SCRIPT} "--cflags" "gssapi" - OUTPUT_VARIABLE _GSS_CFLAGS - RESULT_VARIABLE _GSS_CONFIGURE_FAILED - ) -message(STATUS "CFLAGS: ${_GSS_CFLAGS}") - if(NOT _GSS_CONFIGURE_FAILED) # 0 means success - # should also work in an odd case when multiple directories are given - string(STRIP "${_GSS_CFLAGS}" _GSS_CFLAGS) - string(REGEX REPLACE " +-I" ";" _GSS_CFLAGS "${_GSS_CFLAGS}") - string(REGEX REPLACE " +-([^I][^ \\t;]*)" ";-\\1"_GSS_CFLAGS "${_GSS_CFLAGS}") - - foreach(_flag ${_GSS_CFLAGS}) - if(_flag MATCHES "^-I.*") - string(REGEX REPLACE "^-I" "" _val "${_flag}") - list(APPEND _GSS_INCLUDE_DIR "${_val}") - else() - list(APPEND _GSS_COMPILER_FLAGS "${_flag}") - endif() - endforeach() - endif() + # older versions may not have the "--version" parameter. In this case we just don't care. + if(_GSS_CONFIGURE_FAILED) + set(_GSS_VERSION 0) + endif() - execute_process( - COMMAND ${_GSS_CONFIGURE_SCRIPT} "--libs" "gssapi" - OUTPUT_VARIABLE _GSS_LIB_FLAGS - RESULT_VARIABLE _GSS_CONFIGURE_FAILED - ) -message(STATUS "LDFLAGS: ${_GSS_LIB_FLAGS}") - if(NOT _GSS_CONFIGURE_FAILED) # 0 means success - # this script gives us libraries and link directories. Blah. We have to deal with it. - string(STRIP "${_GSS_LIB_FLAGS}" _GSS_LIB_FLAGS) - string(REGEX REPLACE " +-(L|l)" ";-\\1" _GSS_LIB_FLAGS "${_GSS_LIB_FLAGS}") - string(REGEX REPLACE " +-([^Ll][^ \\t;]*)" ";-\\1"_GSS_LIB_FLAGS "${_GSS_LIB_FLAGS}") - - foreach(_flag ${_GSS_LIB_FLAGS}) - if(_flag MATCHES "^-l.*") - string(REGEX REPLACE "^-l" "" _val "${_flag}") - list(APPEND _GSS_LIBRARIES "${_val}") - elseif(_flag MATCHES "^-L.*") - string(REGEX REPLACE "^-L" "" _val "${_flag}") - list(APPEND _GSS_LINK_DIRECTORIES "${_val}") - else() - list(APPEND _GSS_LINKER_FLAGS "${_flag}") - endif() - endforeach() - endif() + execute_process( + COMMAND ${_GSS_CONFIGURE_SCRIPT} "--vendor" + OUTPUT_VARIABLE _GSS_VENDOR + RESULT_VARIABLE _GSS_CONFIGURE_FAILED + ) + # older versions may not have the "--vendor" parameter. In this case we just don't care. + if(_GSS_CONFIGURE_FAILED) + set(GSS_FLAVOUR "Heimdal") # most probably, shouldn't really matter + else() + if(_GSS_VENDOR MATCHES ".*H|heimdal.*") + set(GSS_FLAVOUR "Heimdal") + else() + set(GSS_FLAVOUR "MIT") + endif() + endif() - execute_process( - COMMAND ${_GSS_CONFIGURE_SCRIPT} "--version" - OUTPUT_VARIABLE _GSS_VERSION - RESULT_VARIABLE _GSS_CONFIGURE_FAILED - ) + else() # either there is no config script or we are on platform that doesn't provide one (Windows?) - # older versions may not have the "--version" parameter. In this case we just don't care. - if(_GSS_CONFIGURE_FAILED) - set(_GSS_VERSION 0) - endif() + find_path(_GSS_INCLUDE_DIR + NAMES + "gssapi/gssapi.h" + HINTS + ${_GSS_ROOT_HINTS} + PATH_SUFFIXES + include + inc + ) + if(_GSS_INCLUDE_DIR) #jay, we've found something + set(CMAKE_REQUIRED_INCLUDES "${_GSS_INCLUDE_DIR}") + check_include_files( "gssapi/gssapi_generic.h;gssapi/gssapi_krb5.h" _GSS_HAVE_MIT_HEADERS) - execute_process( - COMMAND ${_GSS_CONFIGURE_SCRIPT} "--vendor" - OUTPUT_VARIABLE _GSS_VENDOR - RESULT_VARIABLE _GSS_CONFIGURE_FAILED - ) + if(_GSS_HAVE_MIT_HEADERS) + set(GSS_FLAVOUR "MIT") + else() + # prevent compiling the header - just check if we can include it + set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D__ROKEN_H__") + check_include_file( "roken.h" _GSS_HAVE_ROKEN_H) + + check_include_file( "heimdal/roken.h" _GSS_HAVE_HEIMDAL_ROKEN_H) + if(_GSS_HAVE_ROKEN_H OR _GSS_HAVE_HEIMDAL_ROKEN_H) + set(GSS_FLAVOUR "Heimdal") + endif() + set(CMAKE_REQUIRED_DEFINITIONS "") + endif() + else() + # I'm not convienced if this is the right way but this is what autotools do at the moment + find_path(_GSS_INCLUDE_DIR + NAMES + "gssapi.h" + HINTS + ${_GSS_ROOT_HINTS} + PATH_SUFFIXES + include + inc + ) + + if(_GSS_INCLUDE_DIR) + set(GSS_FLAVOUR "Heimdal") + endif() + endif() + + # if we have headers, check if we can link libraries + if(GSS_FLAVOUR) + set(_GSS_LIBDIR_SUFFIXES "") + set(_GSS_LIBDIR_HINTS ${_GSS_ROOT_HINTS}) + get_filename_component(_GSS_CALCULATED_POTENTIAL_ROOT "${_GSS_INCLUDE_DIR}" PATH) + list(APPEND _GSS_LIBDIR_HINTS ${_GSS_CALCULATED_POTENTIAL_ROOT}) - # older versions may not have the "--vendor" parameter. In this case we just don't care. - if(_GSS_CONFIGURE_FAILED) - set(GSS_FLAVOUR "Heimdal") # most probably, shouldn't really matter + if(WIN32) + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + list(APPEND _GSS_LIBDIR_SUFFIXES "lib/AMD64") + if(GSS_FLAVOUR STREQUAL "MIT") + set(_GSS_LIBNAME "gssapi64") + else() + set(_GSS_LIBNAME "libgssapi") + endif() else() - if(_GSS_VENDOR MATCHES ".*H|heimdal.*") - set(GSS_FLAVOUR "Heimdal") - else() - set(GSS_FLAVOUR "MIT") - endif() + list(APPEND _GSS_LIBDIR_SUFFIXES "lib/i386") + if(GSS_FLAVOUR STREQUAL "MIT") + set(_GSS_LIBNAME "gssapi32") + else() + set(_GSS_LIBNAME "libgssapi") + endif() endif() - - else() # either there is no config script or we are on platform that doesn't provide one (Windows?) - - find_path(_GSS_INCLUDE_DIR - NAMES - "gssapi/gssapi.h" - HINTS - ${_GSS_ROOT_HINTS} - PATH_SUFFIXES - include - inc - ) - - if(_GSS_INCLUDE_DIR) #jay, we've found something - set(CMAKE_REQUIRED_INCLUDES "${_GSS_INCLUDE_DIR}") - check_include_files( "gssapi/gssapi_generic.h;gssapi/gssapi_krb5.h" _GSS_HAVE_MIT_HEADERS) - - if(_GSS_HAVE_MIT_HEADERS) - set(GSS_FLAVOUR "MIT") - else() - # prevent compiling the header - just check if we can include it - set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D__ROKEN_H__") - check_include_file( "roken.h" _GSS_HAVE_ROKEN_H) - - check_include_file( "heimdal/roken.h" _GSS_HAVE_HEIMDAL_ROKEN_H) - if(_GSS_HAVE_ROKEN_H OR _GSS_HAVE_HEIMDAL_ROKEN_H) - set(GSS_FLAVOUR "Heimdal") - endif() - set(CMAKE_REQUIRED_DEFINITIONS "") - endif() + else() + list(APPEND _GSS_LIBDIR_SUFFIXES "lib;lib64") # those suffixes are not checked for HINTS + if(GSS_FLAVOUR STREQUAL "MIT") + set(_GSS_LIBNAME "gssapi_krb5") else() - # I'm not convienced if this is the right way but this is what autotools do at the moment - find_path(_GSS_INCLUDE_DIR - NAMES - "gssapi.h" - HINTS - ${_GSS_ROOT_HINTS} - PATH_SUFFIXES - include - inc - ) - - if(_GSS_INCLUDE_DIR) - set(GSS_FLAVOUR "Heimdal") - endif() + set(_GSS_LIBNAME "gssapi") endif() + endif() - # if we have headers, check if we can link libraries - if(GSS_FLAVOUR) - set(_GSS_LIBDIR_SUFFIXES "") - set(_GSS_LIBDIR_HINTS ${_GSS_ROOT_HINTS}) - get_filename_component(_GSS_CALCULATED_POTENTIAL_ROOT "${_GSS_INCLUDE_DIR}" PATH) - list(APPEND _GSS_LIBDIR_HINTS ${_GSS_CALCULATED_POTENTIAL_ROOT}) - - if(WIN32) - if(CMAKE_SIZEOF_VOID_P EQUAL 8) - list(APPEND _GSS_LIBDIR_SUFFIXES "lib/AMD64") - if(GSS_FLAVOUR STREQUAL "MIT") - set(_GSS_LIBNAME "gssapi64") - else() - set(_GSS_LIBNAME "libgssapi") - endif() - else() - list(APPEND _GSS_LIBDIR_SUFFIXES "lib/i386") - if(GSS_FLAVOUR STREQUAL "MIT") - set(_GSS_LIBNAME "gssapi32") - else() - set(_GSS_LIBNAME "libgssapi") - endif() - endif() - else() - list(APPEND _GSS_LIBDIR_SUFFIXES "lib;lib64") # those suffixes are not checked for HINTS - if(GSS_FLAVOUR STREQUAL "MIT") - set(_GSS_LIBNAME "gssapi_krb5") - else() - set(_GSS_LIBNAME "gssapi") - endif() - endif() - - find_library(_GSS_LIBRARIES - NAMES - ${_GSS_LIBNAME} - HINTS - ${_GSS_LIBDIR_HINTS} - PATH_SUFFIXES - ${_GSS_LIBDIR_SUFFIXES} - ) - - endif() + find_library(_GSS_LIBRARIES + NAMES + ${_GSS_LIBNAME} + HINTS + ${_GSS_LIBDIR_HINTS} + PATH_SUFFIXES + ${_GSS_LIBDIR_SUFFIXES} + ) endif() + endif() else() - if(_GSS_PKG_${_MIT_MODNAME}_VERSION) - set(GSS_FLAVOUR "MIT") - set(_GSS_VERSION _GSS_PKG_${_MIT_MODNAME}_VERSION) - else() - set(GSS_FLAVOUR "Heimdal") - set(_GSS_VERSION _GSS_PKG_${_MIT_HEIMDAL}_VERSION) - endif() + if(_GSS_PKG_${_MIT_MODNAME}_VERSION) + set(GSS_FLAVOUR "MIT") + set(_GSS_VERSION _GSS_PKG_${_MIT_MODNAME}_VERSION) + else() + set(GSS_FLAVOUR "Heimdal") + set(_GSS_VERSION _GSS_PKG_${_MIT_HEIMDAL}_VERSION) + endif() endif() set(GSS_INCLUDE_DIR ${_GSS_INCLUDE_DIR}) @@ -243,36 +241,34 @@ set(GSS_COMPILER_FLAGS ${_GSS_COMPILER_FLAGS}) set(GSS_VERSION ${_GSS_VERSION}) if(GSS_FLAVOUR) + if(NOT GSS_VERSION AND GSS_FLAVOUR STREQUAL "Heimdal") + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(HEIMDAL_MANIFEST_FILE "Heimdal.Application.amd64.manifest") + else() + set(HEIMDAL_MANIFEST_FILE "Heimdal.Application.x86.manifest") + endif() - if(NOT GSS_VERSION AND GSS_FLAVOUR STREQUAL "Heimdal") - if(CMAKE_SIZEOF_VOID_P EQUAL 8) - set(HEIMDAL_MANIFEST_FILE "Heimdal.Application.amd64.manifest") - else() - set(HEIMDAL_MANIFEST_FILE "Heimdal.Application.x86.manifest") - endif() - - if(EXISTS "${GSS_INCLUDE_DIR}/${HEIMDAL_MANIFEST_FILE}") - file(STRINGS "${GSS_INCLUDE_DIR}/${HEIMDAL_MANIFEST_FILE}" heimdal_version_str - REGEX "^.*version=\"[0-9]\\.[^\"]+\".*$") + if(EXISTS "${GSS_INCLUDE_DIR}/${HEIMDAL_MANIFEST_FILE}") + file(STRINGS "${GSS_INCLUDE_DIR}/${HEIMDAL_MANIFEST_FILE}" heimdal_version_str + REGEX "^.*version=\"[0-9]\\.[^\"]+\".*$") - string(REGEX MATCH "[0-9]\\.[^\"]+" - GSS_VERSION "${heimdal_version_str}") - endif() + string(REGEX MATCH "[0-9]\\.[^\"]+" + GSS_VERSION "${heimdal_version_str}") + endif() - if(NOT GSS_VERSION) - set(GSS_VERSION "Heimdal Unknown") - endif() - elseif(NOT GSS_VERSION AND GSS_FLAVOUR STREQUAL "MIT") - get_filename_component(_MIT_VERSION "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MIT\\Kerberos\\SDK\\CurrentVersion;VersionString]" NAME CACHE) - if(WIN32 AND _MIT_VERSION) - set(GSS_VERSION "${_MIT_VERSION}") - else() - set(GSS_VERSION "MIT Unknown") - endif() + if(NOT GSS_VERSION) + set(GSS_VERSION "Heimdal Unknown") endif() + elseif(NOT GSS_VERSION AND GSS_FLAVOUR STREQUAL "MIT") + get_filename_component(_MIT_VERSION "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MIT\\Kerberos\\SDK\\CurrentVersion;VersionString]" NAME CACHE) + if(WIN32 AND _MIT_VERSION) + set(GSS_VERSION "${_MIT_VERSION}") + else() + set(GSS_VERSION "MIT Unknown") + endif() + endif() endif() - include(FindPackageHandleStandardArgs) set(_GSS_REQUIRED_VARS GSS_LIBRARIES GSS_FLAVOUR) diff --git a/Utilities/cmcurl/CMake/FindLibSSH2.cmake b/Utilities/cmcurl/CMake/FindLibSSH2.cmake index 12a7c61..84822db 100644 --- a/Utilities/cmcurl/CMake/FindLibSSH2.cmake +++ b/Utilities/cmcurl/CMake/FindLibSSH2.cmake @@ -5,14 +5,14 @@ # LIBSSH2_INCLUDE_DIR - the libssh2 include directory # LIBSSH2_LIBRARY - the libssh2 library name -if (LIBSSH2_INCLUDE_DIR AND LIBSSH2_LIBRARY) +if(LIBSSH2_INCLUDE_DIR AND LIBSSH2_LIBRARY) set(LibSSH2_FIND_QUIETLY TRUE) -endif (LIBSSH2_INCLUDE_DIR AND LIBSSH2_LIBRARY) +endif() -FIND_PATH(LIBSSH2_INCLUDE_DIR libssh2.h +find_path(LIBSSH2_INCLUDE_DIR libssh2.h ) -FIND_LIBRARY(LIBSSH2_LIBRARY NAMES ssh2 +find_library(LIBSSH2_LIBRARY NAMES ssh2 ) if(LIBSSH2_INCLUDE_DIR) @@ -27,9 +27,9 @@ if(LIBSSH2_INCLUDE_DIR) string(REGEX REPLACE "^0(.+)" "\\1" LIBSSH2_VERSION_PATCH "${LIBSSH2_VERSION_PATCH}") set(LIBSSH2_VERSION "${LIBSSH2_VERSION_MAJOR}.${LIBSSH2_VERSION_MINOR}.${LIBSSH2_VERSION_PATCH}") -endif(LIBSSH2_INCLUDE_DIR) +endif() include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibSSH2 DEFAULT_MSG LIBSSH2_INCLUDE_DIR LIBSSH2_LIBRARY ) +find_package_handle_standard_args(LibSSH2 DEFAULT_MSG LIBSSH2_INCLUDE_DIR LIBSSH2_LIBRARY ) -MARK_AS_ADVANCED(LIBSSH2_INCLUDE_DIR LIBSSH2_LIBRARY LIBSSH2_VERSION_MAJOR LIBSSH2_VERSION_MINOR LIBSSH2_VERSION_PATCH LIBSSH2_VERSION) +mark_as_advanced(LIBSSH2_INCLUDE_DIR LIBSSH2_LIBRARY LIBSSH2_VERSION_MAJOR LIBSSH2_VERSION_MINOR LIBSSH2_VERSION_PATCH LIBSSH2_VERSION) diff --git a/Utilities/cmcurl/CMake/FindNGHTTP2.cmake b/Utilities/cmcurl/CMake/FindNGHTTP2.cmake index 4e566cf..348b961 100644 --- a/Utilities/cmcurl/CMake/FindNGHTTP2.cmake +++ b/Utilities/cmcurl/CMake/FindNGHTTP2.cmake @@ -14,5 +14,5 @@ find_package_handle_standard_args(NGHTTP2 "Could NOT find NGHTTP2" ) -set(NGHTTP2_INCLUDE_DIRS ${NGHTTP2_INCLUDE_DIR} ) +set(NGHTTP2_INCLUDE_DIRS ${NGHTTP2_INCLUDE_DIR}) set(NGHTTP2_LIBRARIES ${NGHTTP2_LIBRARY}) diff --git a/Utilities/cmcurl/CMake/Macros.cmake b/Utilities/cmcurl/CMake/Macros.cmake index 82aadca..7f71345 100644 --- a/Utilities/cmcurl/CMake/Macros.cmake +++ b/Utilities/cmcurl/CMake/Macros.cmake @@ -5,35 +5,35 @@ # multiple times with a sequence of possibly dependent libraries in # order of least-to-most-dependent. Some libraries depend on others # to link correctly. -macro(CHECK_LIBRARY_EXISTS_CONCAT LIBRARY SYMBOL VARIABLE) +macro(check_library_exists_concat LIBRARY SYMBOL VARIABLE) check_library_exists("${LIBRARY};${CURL_LIBS}" ${SYMBOL} "${CMAKE_LIBRARY_PATH}" ${VARIABLE}) if(${VARIABLE}) set(CURL_LIBS ${LIBRARY} ${CURL_LIBS}) - endif(${VARIABLE}) -endmacro(CHECK_LIBRARY_EXISTS_CONCAT) + endif() +endmacro() # Check if header file exists and add it to the list. # This macro is intended to be called multiple times with a sequence of # possibly dependent header files. Some headers depend on others to be # compiled correctly. -macro(CHECK_INCLUDE_FILE_CONCAT FILE VARIABLE) +macro(check_include_file_concat FILE VARIABLE) check_include_files("${CURL_INCLUDES};${FILE}" ${VARIABLE}) if(${VARIABLE}) set(CURL_INCLUDES ${CURL_INCLUDES} ${FILE}) set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -D${VARIABLE}") - endif(${VARIABLE}) -endmacro(CHECK_INCLUDE_FILE_CONCAT) + endif() +endmacro() # For other curl specific tests, use this macro. -macro(CURL_INTERNAL_TEST CURL_TEST) +macro(curl_internal_test CURL_TEST) if(NOT DEFINED "${CURL_TEST}") set(MACRO_CHECK_FUNCTION_DEFINITIONS "-D${CURL_TEST} ${CURL_TEST_DEFINES} ${CMAKE_REQUIRED_FLAGS}") if(CMAKE_REQUIRED_LIBRARIES) set(CURL_TEST_ADD_LIBRARIES "-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}") - endif(CMAKE_REQUIRED_LIBRARIES) + endif() message(STATUS "Performing Curl Test ${CURL_TEST}") try_compile(${CURL_TEST} @@ -48,53 +48,17 @@ macro(CURL_INTERNAL_TEST CURL_TEST) file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log "Performing Curl Test ${CURL_TEST} passed with the following output:\n" "${OUTPUT}\n") - else(${CURL_TEST}) + else() message(STATUS "Performing Curl Test ${CURL_TEST} - Failed") set(${CURL_TEST} "" CACHE INTERNAL "Curl test ${FUNCTION}") file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log "Performing Curl Test ${CURL_TEST} failed with the following output:\n" "${OUTPUT}\n") - endif(${CURL_TEST}) - endif() -endmacro(CURL_INTERNAL_TEST) - -macro(CURL_INTERNAL_TEST_RUN CURL_TEST) - if(NOT DEFINED "${CURL_TEST}_COMPILE") - set(MACRO_CHECK_FUNCTION_DEFINITIONS - "-D${CURL_TEST} ${CMAKE_REQUIRED_FLAGS}") - if(CMAKE_REQUIRED_LIBRARIES) - set(CURL_TEST_ADD_LIBRARIES - "-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}") - endif(CMAKE_REQUIRED_LIBRARIES) - - message(STATUS "Performing Curl Test ${CURL_TEST}") - try_run(${CURL_TEST} ${CURL_TEST}_COMPILE - ${CMAKE_BINARY_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/CMake/CurlTests.c - CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS} - "${CURL_TEST_ADD_LIBRARIES}" - OUTPUT_VARIABLE OUTPUT) - if(${CURL_TEST}_COMPILE AND NOT ${CURL_TEST}) - set(${CURL_TEST} 1 CACHE INTERNAL "Curl test ${FUNCTION}") - message(STATUS "Performing Curl Test ${CURL_TEST} - Success") - else(${CURL_TEST}_COMPILE AND NOT ${CURL_TEST}) - message(STATUS "Performing Curl Test ${CURL_TEST} - Failed") - set(${CURL_TEST} "" CACHE INTERNAL "Curl test ${FUNCTION}") - file(APPEND "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log" - "Performing Curl Test ${CURL_TEST} failed with the following output:\n" - "${OUTPUT}") - if(${CURL_TEST}_COMPILE) - file(APPEND - "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log" - "There was a problem running this test\n") - endif(${CURL_TEST}_COMPILE) - file(APPEND "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log" - "\n\n") - endif(${CURL_TEST}_COMPILE AND NOT ${CURL_TEST}) + endif() endif() -endmacro(CURL_INTERNAL_TEST_RUN) +endmacro() -macro(CURL_NROFF_CHECK) +macro(curl_nroff_check) find_program(NROFF NAMES gnroff nroff) if(NROFF) # Need a way to write to stdin, this will do @@ -121,4 +85,4 @@ macro(CURL_NROFF_CHECK) else() message(WARNING "Found no *nroff program") endif() -endmacro(CURL_NROFF_CHECK) +endmacro() diff --git a/Utilities/cmcurl/CMake/OtherTests.cmake b/Utilities/cmcurl/CMake/OtherTests.cmake index 989f04e..ce6d3e1 100644 --- a/Utilities/cmcurl/CMake/OtherTests.cmake +++ b/Utilities/cmcurl/CMake/OtherTests.cmake @@ -5,8 +5,8 @@ set(_source_epilogue "#undef inline") macro(add_header_include check header) if(${check}) set(_source_epilogue "${_source_epilogue}\n#include <${header}>") - endif(${check}) -endmacro(add_header_include) + endif() +endmacro() set(signature_call_conv) if(HAVE_WINDOWS_H) @@ -19,10 +19,10 @@ if(HAVE_WINDOWS_H) if(HAVE_LIBWS2_32) set(CMAKE_REQUIRED_LIBRARIES ws2_32) endif() -else(HAVE_WINDOWS_H) +else() add_header_include(HAVE_SYS_TYPES_H "sys/types.h") add_header_include(HAVE_SYS_SOCKET_H "sys/socket.h") -endif(HAVE_WINDOWS_H) +endif() check_c_source_compiles("${_source_epilogue} int main(void) { @@ -64,13 +64,13 @@ if(curl_cv_recv) set(RECV_TYPE_RETV "${recv_retv}") set(HAVE_RECV 1) set(curl_cv_func_recv_done 1) - endif(curl_cv_func_recv_test) - endif(NOT curl_cv_func_recv_done) - endforeach(recv_arg4) - endforeach(recv_arg3) - endforeach(recv_arg2) - endforeach(recv_arg1) - endforeach(recv_retv) + endif() + endif() + endforeach() + endforeach() + endforeach() + endforeach() + endforeach() else() string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG1 "${curl_cv_func_recv_args}") string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG2 "${curl_cv_func_recv_args}") @@ -81,10 +81,10 @@ if(curl_cv_recv) if("${curl_cv_func_recv_args}" STREQUAL "unknown") message(FATAL_ERROR "Cannot find proper types to use for recv args") - endif("${curl_cv_func_recv_args}" STREQUAL "unknown") -else(curl_cv_recv) + endif() +else() message(FATAL_ERROR "Unable to link function recv") -endif(curl_cv_recv) +endif() set(curl_cv_func_recv_args "${curl_cv_func_recv_args}" CACHE INTERNAL "Arguments for recv") set(HAVE_RECV 1) @@ -130,13 +130,13 @@ if(curl_cv_send) set(SEND_TYPE_RETV "${send_retv}") set(HAVE_SEND 1) set(curl_cv_func_send_done 1) - endif(curl_cv_func_send_test) - endif(NOT curl_cv_func_send_done) - endforeach(send_arg4) - endforeach(send_arg3) - endforeach(send_arg2) - endforeach(send_arg1) - endforeach(send_retv) + endif() + endif() + endforeach() + endforeach() + endforeach() + endforeach() + endforeach() else() string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG1 "${curl_cv_func_send_args}") string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG2 "${curl_cv_func_send_args}") @@ -148,11 +148,11 @@ if(curl_cv_send) if("${curl_cv_func_send_args}" STREQUAL "unknown") message(FATAL_ERROR "Cannot find proper types to use for send args") - endif("${curl_cv_func_send_args}" STREQUAL "unknown") + endif() set(SEND_QUAL_ARG2 "const") -else(curl_cv_send) +else() message(FATAL_ERROR "Unable to link function send") -endif(curl_cv_send) +endif() set(curl_cv_func_send_args "${curl_cv_func_send_args}" CACHE INTERNAL "Arguments for send") set(HAVE_SEND 1) @@ -184,7 +184,7 @@ if(NOT APPLE) set(CMAKE_REQUIRED_FLAGS) if(HAVE_SYS_POLL_H) set(CMAKE_REQUIRED_FLAGS "-DHAVE_SYS_POLL_H") - endif(HAVE_SYS_POLL_H) + endif() check_c_source_runs(" #ifdef HAVE_SYS_POLL_H # include <sys/poll.h> @@ -199,7 +199,7 @@ set(CMAKE_REQUIRED_FLAGS) if(HAVE_SIGNAL_H) set(CMAKE_REQUIRED_FLAGS "-DHAVE_SIGNAL_H") set(CMAKE_EXTRA_INCLUDE_FILES "signal.h") -endif(HAVE_SIGNAL_H) +endif() check_type_size("sig_atomic_t" SIZEOF_SIG_ATOMIC_T) if(HAVE_SIZEOF_SIG_ATOMIC_T) check_c_source_compiles(" @@ -213,8 +213,8 @@ if(HAVE_SIZEOF_SIG_ATOMIC_T) }" HAVE_SIG_ATOMIC_T_NOT_VOLATILE) if(NOT HAVE_SIG_ATOMIC_T_NOT_VOLATILE) set(HAVE_SIG_ATOMIC_T_VOLATILE 1) - endif(NOT HAVE_SIG_ATOMIC_T_NOT_VOLATILE) -endif(HAVE_SIZEOF_SIG_ATOMIC_T) + endif() +endif() if(HAVE_WINDOWS_H) set(CMAKE_EXTRA_INCLUDE_FILES winsock2.h) @@ -222,11 +222,10 @@ else() set(CMAKE_EXTRA_INCLUDE_FILES) if(HAVE_SYS_SOCKET_H) set(CMAKE_EXTRA_INCLUDE_FILES sys/socket.h) - endif(HAVE_SYS_SOCKET_H) + endif() endif() check_type_size("struct sockaddr_storage" SIZEOF_STRUCT_SOCKADDR_STORAGE) if(HAVE_SIZEOF_STRUCT_SOCKADDR_STORAGE) set(HAVE_STRUCT_SOCKADDR_STORAGE 1) -endif(HAVE_SIZEOF_STRUCT_SOCKADDR_STORAGE) - +endif() diff --git a/Utilities/cmcurl/CMake/Platforms/WindowsCache.cmake b/Utilities/cmcurl/CMake/Platforms/WindowsCache.cmake index 53d0a5e..2dbe1bb 100644 --- a/Utilities/cmcurl/CMake/Platforms/WindowsCache.cmake +++ b/Utilities/cmcurl/CMake/Platforms/WindowsCache.cmake @@ -115,8 +115,7 @@ if(NOT UNIX) set(HAVE_SIGACTION 0) set(HAVE_MACRO_SIGSETJMP 0) - else(WIN32) + else() message("This file should be included on Windows platform only") - endif(WIN32) -endif(NOT UNIX) - + endif() +endif() diff --git a/Utilities/cmcurl/CMake/Utilities.cmake b/Utilities/cmcurl/CMake/Utilities.cmake index 005b166..5cb1d44 100644 --- a/Utilities/cmcurl/CMake/Utilities.cmake +++ b/Utilities/cmcurl/CMake/Utilities.cmake @@ -1,44 +1,13 @@ # File containing various utilities -# Converts a CMake list to a string containing elements separated by spaces -function(TO_LIST_SPACES _LIST_NAME OUTPUT_VAR) - set(NEW_LIST_SPACE) - foreach(ITEM ${${_LIST_NAME}}) - set(NEW_LIST_SPACE "${NEW_LIST_SPACE} ${ITEM}") - endforeach() - string(STRIP ${NEW_LIST_SPACE} NEW_LIST_SPACE) - set(${OUTPUT_VAR} "${NEW_LIST_SPACE}" PARENT_SCOPE) -endfunction() - -# Appends a lis of item to a string which is a space-separated list, if they don't already exist. -function(LIST_SPACES_APPEND_ONCE LIST_NAME) - string(REPLACE " " ";" _LIST ${${LIST_NAME}}) - list(APPEND _LIST ${ARGN}) - list(REMOVE_DUPLICATES _LIST) - to_list_spaces(_LIST NEW_LIST_SPACE) - set(${LIST_NAME} "${NEW_LIST_SPACE}" PARENT_SCOPE) -endfunction() - -# Convenience function that does the same as LIST(FIND ...) but with a TRUE/FALSE return value. -# Ex: IN_STR_LIST(MY_LIST "Searched item" WAS_FOUND) -function(IN_STR_LIST LIST_NAME ITEM_SEARCHED RETVAL) - list(FIND ${LIST_NAME} ${ITEM_SEARCHED} FIND_POS) - if(${FIND_POS} EQUAL -1) - set(${RETVAL} FALSE PARENT_SCOPE) - else() - set(${RETVAL} TRUE PARENT_SCOPE) - endif() -endfunction() - # Returns a list of arguments that evaluate to true -function(collect_true output_var output_count_var) - set(${output_var}) +function(count_true output_count_var) + set(lst) foreach(option_var IN LISTS ARGN) if(${option_var}) - list(APPEND ${output_var} ${option_var}) + list(APPEND lst ${option_var}) endif() endforeach() - set(${output_var} ${${output_var}} PARENT_SCOPE) - list(LENGTH ${output_var} ${output_count_var}) - set(${output_count_var} ${${output_count_var}} PARENT_SCOPE) + list(LENGTH lst lst_len) + set(${output_count_var} ${lst_len} PARENT_SCOPE) endfunction() diff --git a/Utilities/cmcurl/CMake/cmake_uninstall.cmake.in b/Utilities/cmcurl/CMake/cmake_uninstall.cmake.in index d00a516..db8e536 100644 --- a/Utilities/cmcurl/CMake/cmake_uninstall.cmake.in +++ b/Utilities/cmcurl/CMake/cmake_uninstall.cmake.in @@ -1,11 +1,11 @@ if(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") message(FATAL_ERROR "Cannot find install manifest: @CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") -endif(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") +endif() -if (NOT DEFINED CMAKE_INSTALL_PREFIX) - set (CMAKE_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@") -endif () - message(${CMAKE_INSTALL_PREFIX}) +if(NOT DEFINED CMAKE_INSTALL_PREFIX) + set(CMAKE_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@") +endif() +message(${CMAKE_INSTALL_PREFIX}) file(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files) string(REGEX REPLACE "\n" ";" files "${files}") @@ -19,8 +19,8 @@ foreach(file ${files}) ) if(NOT "${rm_retval}" STREQUAL 0) message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}") - endif(NOT "${rm_retval}" STREQUAL 0) - else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") + endif() + else() message(STATUS "File $ENV{DESTDIR}${file} does not exist.") - endif(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") -endforeach(file) + endif() +endforeach() diff --git a/Utilities/cmcurl/CMake/curl-config.cmake b/Utilities/cmcurl/CMake/curl-config.cmake deleted file mode 100644 index 119332c..0000000 --- a/Utilities/cmcurl/CMake/curl-config.cmake +++ /dev/null @@ -1,59 +0,0 @@ - -get_filename_component(_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) - -if(NOT CURL_FIND_COMPONENTS) - set(CURL_FIND_COMPONENTS curl libcurl) - if(CURL_FIND_REQUIRED) - set(CURL_FIND_REQUIRED_curl TRUE) - set(CURL_FIND_REQUIRED_libcurl TRUE) - endif() -endif() - -set(_curl_missing_components) -foreach(_comp ${CURL_FIND_COMPONENTS}) - if(EXISTS "${_DIR}/${_comp}-target.cmake") - include("${_DIR}/${_comp}-target.cmake") - set(CURL_${_comp}_FOUND TRUE) - else() - set(CURL_${_comp}_FOUND FALSE) - if(CURL_FIND_REQUIRED_${_comp}) - set(CURL_FOUND FALSE) - list(APPEND _curl_missing_components ${_comp}) - endif() - endif() -endforeach() - -if(_curl_missing_components) - set(CURL_NOT_FOUND_MESSAGE "Following required components not found: " ${_curl_missing_components}) -else() - if(TARGET CURL::libcurl) - string(TOUPPER "${CMAKE_BUILD_TYPE}" _curl_current_config) - if(NOT _curl_current_config) - set(_curl_current_config "NOCONFIG") - endif() - get_target_property(_curl_configurations CURL::libcurl IMPORTED_CONFIGURATIONS) - list(FIND _curl_configurations "${_curl_current_config}" _i) - if(_i LESS 0) - set(_curl_config "RELEASE") - list(FIND _curl_configurations "${_curl_current_config}" _i) - if(_i LESS 0) - set(_curl_config "NOCONFIG") - list(FIND _curl_configurations "${_curl_current_config}" _i) - endif() - endif() - - if(_i LESS 0) - set(_curl_current_config "") # let CMake pick config at random - else() - set(_curl_current_config "_${_curl_current_config}") - endif() - - get_target_property(CURL_INCLUDE_DIRS CURL::libcurl INTERFACE_INCLUDE_DIRECTORIES) - get_target_property(CURL_LIBRARIES CURL::libcurl "LOCATION${_curl_current_config}") - set(_curl_current_config) - set(_curl_configurations) - set(_i) - endif() -endif() - -unset(_curl_missing_components) diff --git a/Utilities/cmcurl/CMake/curl-config.cmake.in b/Utilities/cmcurl/CMake/curl-config.cmake.in new file mode 100644 index 0000000..73e04c6 --- /dev/null +++ b/Utilities/cmcurl/CMake/curl-config.cmake.in @@ -0,0 +1,64 @@ + +get_filename_component(_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) + +if(NOT CURL_FIND_COMPONENTS) + set(CURL_FIND_COMPONENTS curl libcurl) + if(CURL_FIND_REQUIRED) + set(CURL_FIND_REQUIRED_curl TRUE) + set(CURL_FIND_REQUIRED_libcurl TRUE) + endif() +endif() + +include(CMakeFindDependencyMacro) +if(CURL_FIND_REQUIRED_libcurl) + find_dependency(OpenSSL "@OPENSSL_VERSION_MAJOR@") +endif() + +set(_curl_missing_components) +foreach(_comp ${CURL_FIND_COMPONENTS}) + if(EXISTS "${_DIR}/${_comp}-target.cmake") + include("${_DIR}/${_comp}-target.cmake") + set(CURL_${_comp}_FOUND TRUE) + else() + set(CURL_${_comp}_FOUND FALSE) + if(CURL_FIND_REQUIRED_${_comp}) + set(CURL_FOUND FALSE) + list(APPEND _curl_missing_components ${_comp}) + endif() + endif() +endforeach() + +if(_curl_missing_components) + set(CURL_NOT_FOUND_MESSAGE "Following required components not found: " ${_curl_missing_components}) +else() + if(TARGET CURL::libcurl) + string(TOUPPER "${CMAKE_BUILD_TYPE}" _curl_current_config) + if(NOT _curl_current_config) + set(_curl_current_config "NOCONFIG") + endif() + get_target_property(_curl_configurations CURL::libcurl IMPORTED_CONFIGURATIONS) + list(FIND _curl_configurations "${_curl_current_config}" _i) + if(_i LESS 0) + set(_curl_config "RELEASE") + list(FIND _curl_configurations "${_curl_current_config}" _i) + if(_i LESS 0) + set(_curl_config "NOCONFIG") + list(FIND _curl_configurations "${_curl_current_config}" _i) + endif() + endif() + + if(_i LESS 0) + set(_curl_current_config "") # let CMake pick config at random + else() + set(_curl_current_config "_${_curl_current_config}") + endif() + + get_target_property(CURL_INCLUDE_DIRS CURL::libcurl INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(CURL_LIBRARIES CURL::libcurl "LOCATION${_curl_current_config}") + set(_curl_current_config) + set(_curl_configurations) + set(_i) + endif() +endif() + +unset(_curl_missing_components) diff --git a/Utilities/cmcurl/CMakeLists.txt b/Utilities/cmcurl/CMakeLists.txt index 0128214..636530e 100644 --- a/Utilities/cmcurl/CMakeLists.txt +++ b/Utilities/cmcurl/CMakeLists.txt @@ -2,6 +2,7 @@ set(BUILD_CURL_EXE OFF CACHE INTERNAL "No curl exe") set(BUILD_DASHBOARD_REPORTS OFF CACHE INTERNAL "No curl dashboard reports") set(BUILD_RELEASE_DEBUG_DIRS OFF CACHE INTERNAL "No curl release/debug dirs") +set(BUILD_SHARED_LIBS OFF CACHE INTERNAL "Build shared libraries") set(CMAKE_USE_GSSAPI OFF CACHE INTERNAL "Disable curl gssapi") set(CMAKE_USE_LIBSSH2 OFF CACHE INTERNAL "Disable curl libssh2") set(CMAKE_USE_OPENLDAP OFF CACHE INTERNAL "No curl OpenLDAP") @@ -23,7 +24,6 @@ set(CURL_DISABLE_TELNET ON CACHE INTERNAL "Disable curl telnet protocol?") set(CURL_DISABLE_TFTP ON CACHE INTERNAL "Disable curl tftp protocol?") set(CURL_DISABLE_VERBOSE_STRINGS OFF CACHE INTERNAL "Do not disable curl verbosity") set(CURL_HIDDEN_SYMBOLS OFF CACHE INTERNAL "No curl hidden symbols") -set(CURL_STATICLIB ON CACHE INTERNAL "Static curl") set(CURL_WERROR OFF CACHE INTERNAL "Turn compiler warnings into errors") set(DISABLED_THREADSAFE OFF CACHE INTERNAL "Curl can use thread-safe functions") set(ENABLE_ARES OFF CACHE INTERNAL "No curl c-ares support") @@ -123,26 +123,26 @@ endif() # To check: # (From Daniel Stenberg) The cmake build selected to run gcc with -fPIC on my box while the plain configure script did not. # (From Daniel Stenberg) The gcc command line use neither -g nor any -O options. As a developer, I also treasure our configure scripts's --enable-debug option that sets a long range of "picky" compiler options. -cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}") include(Utilities) include(Macros) include(CMakeDependentOption) include(CheckCCompilerFlag) -project( CURL C ) +project(CURL C) if(0) # This code not needed for building within CMake. message(WARNING "the curl cmake build system is poorly maintained. Be aware") endif() -file (READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS) -string (REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*" +file(READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS) +string(REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*" CURL_VERSION ${CURL_VERSION_H_CONTENTS}) -string (REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION}) -string (REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+" +string(REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION}) +string(REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+" CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS}) -string (REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM}) +string(REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM}) include_regular_expression("^.*$") # Sukender: Is it necessary? @@ -160,19 +160,19 @@ set(OPERATING_SYSTEM "${CMAKE_SYSTEM_NAME}") set(OS "\"${CMAKE_SYSTEM_NAME}\"") include_directories(${PROJECT_BINARY_DIR}/include/curl) -include_directories( ${CURL_SOURCE_DIR}/include ) +include_directories(${CURL_SOURCE_DIR}/include) option(CURL_WERROR "Turn compiler warnings into errors" OFF) option(PICKY_COMPILER "Enable picky compiler options" ON) option(BUILD_CURL_EXE "Set to ON to build curl executable." ON) -option(CURL_STATICLIB "Set to ON to build libcurl with static linking." OFF) +option(BUILD_SHARED_LIBS "Build shared libraries" ON) option(ENABLE_ARES "Set to ON to enable c-ares support" OFF) if(WIN32) option(CURL_STATIC_CRT "Set to ON to build libcurl with static CRT on Windows (/MT)." OFF) option(ENABLE_INET_PTON "Set to OFF to prevent usage of inet_pton when building against modern SDKs while still requiring compatibility with older Windows versions, such as Windows XP, Windows Server 2003 etc." ON) endif() -CMAKE_DEPENDENT_OPTION(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" +cmake_dependent_option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" ON "NOT ENABLE_ARES" OFF) @@ -180,19 +180,19 @@ option(ENABLE_DEBUG "Set to ON to enable curl debug features" OFF) option(ENABLE_CURLDEBUG "Set to ON to build with TrackMemory feature enabled" OFF) if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) - if (PICKY_COMPILER) - foreach (_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers) + if(PICKY_COMPILER) + foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers) # surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new # test result in. - CHECK_C_COMPILER_FLAG(${_CCOPT} OPT${_CCOPT}) + check_c_compiler_flag(${_CCOPT} OPT${_CCOPT}) if(OPT${_CCOPT}) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}") endif() endforeach() - endif(PICKY_COMPILER) -endif(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) + endif() +endif() -if (ENABLE_DEBUG) +if(ENABLE_DEBUG) # DEBUGBUILD will be defined only for Debug builds if(NOT CMAKE_VERSION VERSION_LESS 3.0) set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS $<$<CONFIG:Debug>:DEBUGBUILD>) @@ -202,13 +202,15 @@ if (ENABLE_DEBUG) set(ENABLE_CURLDEBUG ON) endif() -if (ENABLE_CURLDEBUG) +if(ENABLE_CURLDEBUG) set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS CURLDEBUG) endif() if(0) # This code not needed for building within CMake. # For debug libs and exes, add "-d" postfix -set(CMAKE_DEBUG_POSTFIX "-d" CACHE STRING "Set debug library postfix") +if(NOT DEFINED CMAKE_DEBUG_POSTFIX) + set(CMAKE_DEBUG_POSTFIX "-d") +endif() endif() # initialize CURL_LIBS @@ -217,7 +219,7 @@ set(CURL_LIBS "") if(ENABLE_ARES) set(USE_ARES 1) find_package(CARES REQUIRED) - list(APPEND CURL_LIBS ${CARES_LIBRARY} ) + list(APPEND CURL_LIBS ${CARES_LIBRARY}) set(CURL_LIBS ${CURL_LIBS} ${CARES_LIBRARY}) endif() @@ -297,11 +299,11 @@ if(ENABLE_IPV6 AND NOT WIN32) endif() if(0) # This code not needed for building within CMake. -CURL_NROFF_CHECK() # Required for building manual, docs, tests +curl_nroff_check() find_package(Perl) -CMAKE_DEPENDENT_OPTION(ENABLE_MANUAL "to provide the built-in manual" +cmake_dependent_option(ENABLE_MANUAL "to provide the built-in manual" ON "NROFF_USEFUL;PERL_FOUND" OFF) @@ -326,27 +328,27 @@ endif() # Disable warnings on Borland to avoid changing 3rd party code. if(BORLAND) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w-") -endif(BORLAND) +endif() # If we are on AIX, do the _ALL_SOURCE magic if(${CMAKE_SYSTEM_NAME} MATCHES AIX) set(_ALL_SOURCE 1) -endif(${CMAKE_SYSTEM_NAME} MATCHES AIX) +endif() # Include all the necessary files for macros -include (CheckFunctionExists) -include (CheckIncludeFile) -include (CheckIncludeFiles) -include (CheckLibraryExists) -include (CheckSymbolExists) -include (CheckTypeSize) -include (CheckCSourceCompiles) +include(CheckFunctionExists) +include(CheckIncludeFile) +include(CheckIncludeFiles) +include(CheckLibraryExists) +include(CheckSymbolExists) +include(CheckTypeSize) +include(CheckCSourceCompiles) # On windows preload settings if(WIN32) set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D_WINSOCKAPI_=") include(${CMAKE_CURRENT_SOURCE_DIR}/CMake/Platforms/WindowsCache.cmake) -endif(WIN32) +endif() if(ENABLE_THREADED_RESOLVER) find_package(Threads REQUIRED) @@ -375,13 +377,13 @@ if(BEOS) set(NOT_NEED_LIBNSL 1) check_library_exists_concat("bind" gethostbyname HAVE_LIBBIND) check_library_exists_concat("bnetapi" closesocket HAVE_LIBBNETAPI) -endif(BEOS) +endif() check_library_exists_concat("network" recv HAVE_LIBNETWORK) if(NOT NOT_NEED_LIBNSL) check_library_exists_concat("nsl" gethostbyname HAVE_LIBNSL) -endif(NOT NOT_NEED_LIBNSL) +endif() check_function_exists(gethostname HAVE_GETHOSTNAME) @@ -409,14 +411,14 @@ if(WIN32 OR CMAKE_USE_DARWINSSL OR CMAKE_USE_WINSSL OR CMAKE_USE_MBEDTLS) set(openssl_default OFF) endif() -collect_true(enabled_ssl_options enabled_ssl_options_count +count_true(enabled_ssl_options_count CMAKE_USE_WINSSL CMAKE_USE_DARWINSSL CMAKE_USE_OPENSSL CMAKE_USE_MBEDTLS ) -if(enabled_ssl_options_count GREATER 1) - message(FATAL_ERROR "Multiple SSL options specified: ${enabled_ssl_options}. Please pick at most one and disable the rest.") +if(enabled_ssl_options_count GREATER "1") + set(CURL_WITH_MULTI_SSL ON) endif() if(CMAKE_USE_WINSSL) @@ -454,9 +456,9 @@ if(CMAKE_USE_OPENSSL) set(HAVE_LIBSSL ON) list(APPEND CURL_LIBS ${OPENSSL_LIBRARIES}) include_directories(${OPENSSL_INCLUDE_DIR}) + set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR}) check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H) - check_include_file("openssl/engine.h" HAVE_OPENSSL_ENGINE_H) check_include_file("openssl/err.h" HAVE_OPENSSL_ERR_H) check_include_file("openssl/pem.h" HAVE_OPENSSL_PEM_H) check_include_file("openssl/rsa.h" HAVE_OPENSSL_RSA_H) @@ -585,7 +587,6 @@ if(NOT CURL_DISABLE_LDAP) endif() endif() endif() - endif() # No ldap, no ldaps. @@ -677,9 +678,8 @@ if(CMAKE_USE_LIBSSH2) check_function_exists(libssh2_scp_send64 HAVE_LIBSSH2_SCP_SEND64) check_function_exists(libssh2_session_handshake HAVE_LIBSSH2_SESSION_HANDSHAKE) set(CMAKE_EXTRA_INCLUDE_FILES "") - - endif(LIBSSH2_FOUND) -endif(CMAKE_USE_LIBSSH2) + endif() +endif() option(CMAKE_USE_GSSAPI "Use GSSAPI implementation (right now only Heimdal is supported with CMake build)" OFF) mark_as_advanced(CMAKE_USE_GSSAPI) @@ -761,68 +761,67 @@ set(CURL_CA_PATH "auto" CACHE STRING "Location of default CA path. Set 'none' to disable or 'auto' for auto-detection. Defaults to 'auto'.") if("${CURL_CA_BUNDLE}" STREQUAL "") - message(FATAL_ERROR "Invalid value of CURL_CA_BUNDLE. Use 'none', 'auto' or file path.") + message(FATAL_ERROR "Invalid value of CURL_CA_BUNDLE. Use 'none', 'auto' or file path.") elseif("${CURL_CA_BUNDLE}" STREQUAL "none") - unset(CURL_CA_BUNDLE CACHE) + unset(CURL_CA_BUNDLE CACHE) elseif("${CURL_CA_BUNDLE}" STREQUAL "auto") - unset(CURL_CA_BUNDLE CACHE) - set(CURL_CA_BUNDLE_AUTODETECT TRUE) + unset(CURL_CA_BUNDLE CACHE) + set(CURL_CA_BUNDLE_AUTODETECT TRUE) else() - set(CURL_CA_BUNDLE_SET TRUE) + set(CURL_CA_BUNDLE_SET TRUE) endif() if("${CURL_CA_PATH}" STREQUAL "") - message(FATAL_ERROR "Invalid value of CURL_CA_PATH. Use 'none', 'auto' or directory path.") + message(FATAL_ERROR "Invalid value of CURL_CA_PATH. Use 'none', 'auto' or directory path.") elseif("${CURL_CA_PATH}" STREQUAL "none") - unset(CURL_CA_PATH CACHE) + unset(CURL_CA_PATH CACHE) elseif("${CURL_CA_PATH}" STREQUAL "auto") - unset(CURL_CA_PATH CACHE) - set(CURL_CA_PATH_AUTODETECT TRUE) + unset(CURL_CA_PATH CACHE) + set(CURL_CA_PATH_AUTODETECT TRUE) else() - set(CURL_CA_PATH_SET TRUE) + set(CURL_CA_PATH_SET TRUE) endif() if(CURL_CA_BUNDLE_SET AND CURL_CA_PATH_AUTODETECT) - # Skip autodetection of unset CA path because CA bundle is set explicitly + # Skip autodetection of unset CA path because CA bundle is set explicitly elseif(CURL_CA_PATH_SET AND CURL_CA_BUNDLE_AUTODETECT) - # Skip autodetection of unset CA bundle because CA path is set explicitly + # Skip autodetection of unset CA bundle because CA path is set explicitly elseif(CURL_CA_PATH_AUTODETECT OR CURL_CA_BUNDLE_AUTODETECT) - # first try autodetecting a CA bundle, then a CA path - - if(CURL_CA_BUNDLE_AUTODETECT) - set(SEARCH_CA_BUNDLE_PATHS - /etc/ssl/certs/ca-certificates.crt - /etc/pki/tls/certs/ca-bundle.crt - /usr/share/ssl/certs/ca-bundle.crt - /usr/local/share/certs/ca-root-nss.crt - /etc/ssl/cert.pem) - - foreach(SEARCH_CA_BUNDLE_PATH ${SEARCH_CA_BUNDLE_PATHS}) - if(EXISTS "${SEARCH_CA_BUNDLE_PATH}") - message(STATUS "Found CA bundle: ${SEARCH_CA_BUNDLE_PATH}") - set(CURL_CA_BUNDLE "${SEARCH_CA_BUNDLE_PATH}") - set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set") - break() - endif() - endforeach() - endif() + # first try autodetecting a CA bundle, then a CA path + + if(CURL_CA_BUNDLE_AUTODETECT) + set(SEARCH_CA_BUNDLE_PATHS + /etc/ssl/certs/ca-certificates.crt + /etc/pki/tls/certs/ca-bundle.crt + /usr/share/ssl/certs/ca-bundle.crt + /usr/local/share/certs/ca-root-nss.crt + /etc/ssl/cert.pem) + + foreach(SEARCH_CA_BUNDLE_PATH ${SEARCH_CA_BUNDLE_PATHS}) + if(EXISTS "${SEARCH_CA_BUNDLE_PATH}") + message(STATUS "Found CA bundle: ${SEARCH_CA_BUNDLE_PATH}") + set(CURL_CA_BUNDLE "${SEARCH_CA_BUNDLE_PATH}") + set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set") + break() + endif() + endforeach() + endif() - if(CURL_CA_PATH_AUTODETECT AND (NOT CURL_CA_PATH_SET)) - if(EXISTS "/etc/ssl/certs") - set(CURL_CA_PATH "/etc/ssl/certs") - set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set") - endif() + if(CURL_CA_PATH_AUTODETECT AND (NOT CURL_CA_PATH_SET)) + if(EXISTS "/etc/ssl/certs") + set(CURL_CA_PATH "/etc/ssl/certs") + set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set") endif() + endif() endif() if(CURL_CA_PATH_SET AND NOT USE_OPENSSL AND NOT USE_MBEDTLS) - message(FATAL_ERROR - "CA path only supported by OpenSSL, GnuTLS or mbed TLS. " - "Set CURL_CA_PATH=none or enable one of those TLS backends.") + message(FATAL_ERROR + "CA path only supported by OpenSSL, GnuTLS or mbed TLS. " + "Set CURL_CA_PATH=none or enable one of those TLS backends.") endif() endif() - # Check for header files if(NOT UNIX) check_include_file_concat("windows.h" HAVE_WINDOWS_H) @@ -920,7 +919,7 @@ check_type_size("time_t" SIZEOF_TIME_T) if(HAVE_SIZEOF_LONG_LONG) set(HAVE_LONGLONG 1) set(HAVE_LL 1) -endif(HAVE_SIZEOF_LONG_LONG) +endif() find_file(RANDOM_FILE urandom /dev) mark_as_advanced(RANDOM_FILE) @@ -954,7 +953,7 @@ check_symbol_exists(strncmpi "${CURL_INCLUDES}" HAVE_STRNCMPI) check_symbol_exists(alarm "${CURL_INCLUDES}" HAVE_ALARM) if(NOT HAVE_STRNCMPI) set(HAVE_STRCMPI) -endif(NOT HAVE_STRNCMPI) +endif() check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR) check_symbol_exists(gethostbyaddr_r "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR_R) check_symbol_exists(gettimeofday "${CURL_INCLUDES}" HAVE_GETTIMEOFDAY) @@ -970,6 +969,7 @@ check_symbol_exists(sigsetjmp "${CURL_INCLUDES}" HAVE_SIGSETJMP) check_symbol_exists(getpass_r "${CURL_INCLUDES}" HAVE_GETPASS_R) check_symbol_exists(strlcat "${CURL_INCLUDES}" HAVE_STRLCAT) check_symbol_exists(getpwuid "${CURL_INCLUDES}" HAVE_GETPWUID) +check_symbol_exists(getpwuid_r "${CURL_INCLUDES}" HAVE_GETPWUID_R) check_symbol_exists(geteuid "${CURL_INCLUDES}" HAVE_GETEUID) check_symbol_exists(utime "${CURL_INCLUDES}" HAVE_UTIME) check_symbol_exists(gmtime_r "${CURL_INCLUDES}" HAVE_GMTIME_R) @@ -982,7 +982,7 @@ check_symbol_exists(signal "${CURL_INCLUDES}" HAVE_SIGNAL_FUNC) check_symbol_exists(SIGALRM "${CURL_INCLUDES}" HAVE_SIGNAL_MACRO) if(HAVE_SIGNAL_FUNC AND HAVE_SIGNAL_MACRO) set(HAVE_SIGNAL 1) -endif(HAVE_SIGNAL_FUNC AND HAVE_SIGNAL_MACRO) +endif() check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME) check_symbol_exists(strtoll "${CURL_INCLUDES}" HAVE_STRTOLL) check_symbol_exists(_strtoi64 "${CURL_INCLUDES}" HAVE__STRTOI64) @@ -1016,35 +1016,35 @@ if(WIN32) add_definitions(-D_WIN32_WINNT=0x0501) endif() else() - check_function_exists(inet_pton HAVE_INET_PTON) + check_function_exists(inet_pton HAVE_INET_PTON) endif() check_symbol_exists(fsetxattr "${CURL_INCLUDES}" HAVE_FSETXATTR) if(HAVE_FSETXATTR) foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6) - curl_internal_test_run(${CURL_TEST}) - endforeach(CURL_TEST) -endif(HAVE_FSETXATTR) + curl_internal_test(${CURL_TEST}) + endforeach() +endif() # sigaction and sigsetjmp are special. Use special mechanism for # detecting those, but only if previous attempt failed. if(HAVE_SIGNAL_H) check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION) -endif(HAVE_SIGNAL_H) +endif() if(NOT HAVE_SIGSETJMP) if(HAVE_SETJMP_H) check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP) if(HAVE_MACRO_SIGSETJMP) set(HAVE_SIGSETJMP 1) - endif(HAVE_MACRO_SIGSETJMP) - endif(HAVE_SETJMP_H) -endif(NOT HAVE_SIGSETJMP) + endif() + endif() +endif() # If there is no stricmp(), do not allow LDAP to parse URLs if(NOT HAVE_STRICMP) set(HAVE_LDAP_URL_PARSE 1) -endif(NOT HAVE_STRICMP) +endif() # Do curl specific tests foreach(CURL_TEST @@ -1082,12 +1082,12 @@ foreach(CURL_TEST HAVE_FILE_OFFSET_BITS ) curl_internal_test(${CURL_TEST}) -endforeach(CURL_TEST) +endforeach() if(HAVE_FILE_OFFSET_BITS) set(_FILE_OFFSET_BITS 64) set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64") -endif(HAVE_FILE_OFFSET_BITS) +endif() check_type_size("off_t" SIZEOF_OFF_T) # include this header to get the type @@ -1102,8 +1102,8 @@ foreach(CURL_TEST HAVE_GLIBC_STRERROR_R HAVE_POSIX_STRERROR_R ) - curl_internal_test_run(${CURL_TEST}) -endforeach(CURL_TEST) + curl_internal_test(${CURL_TEST}) +endforeach() # Check for reentrant foreach(CURL_TEST @@ -1117,9 +1117,9 @@ foreach(CURL_TEST if(NOT ${CURL_TEST}) if(${CURL_TEST}_REENTRANT) set(NEED_REENTRANT 1) - endif(${CURL_TEST}_REENTRANT) - endif(NOT ${CURL_TEST}) -endforeach(CURL_TEST) + endif() + endif() +endforeach() if(NEED_REENTRANT) foreach(CURL_TEST @@ -1132,32 +1132,32 @@ if(NEED_REENTRANT) set(${CURL_TEST} 0) if(${CURL_TEST}_REENTRANT) set(${CURL_TEST} 1) - endif(${CURL_TEST}_REENTRANT) - endforeach(CURL_TEST) -endif(NEED_REENTRANT) + endif() + endforeach() +endif() if(HAVE_INET_NTOA_R_DECL_REENTRANT) set(HAVE_INET_NTOA_R_DECL 1) set(NEED_REENTRANT 1) -endif(HAVE_INET_NTOA_R_DECL_REENTRANT) +endif() # Some other minor tests if(NOT HAVE_IN_ADDR_T) set(in_addr_t "unsigned long") -endif(NOT HAVE_IN_ADDR_T) +endif() # Fix libz / zlib.h if(NOT CURL_SPECIAL_LIBZ) if(NOT HAVE_LIBZ) set(HAVE_ZLIB_H 0) - endif(NOT HAVE_LIBZ) + endif() if(NOT HAVE_ZLIB_H) set(HAVE_LIBZ 0) - endif(NOT HAVE_ZLIB_H) -endif(NOT CURL_SPECIAL_LIBZ) + endif() +endif() # Check for nonblocking set(HAVE_DISABLED_NONBLOCKING 1) @@ -1166,16 +1166,13 @@ if(HAVE_FIONBIO OR HAVE_IOCTLSOCKET_CASE OR HAVE_O_NONBLOCK) set(HAVE_DISABLED_NONBLOCKING) -endif(HAVE_FIONBIO OR - HAVE_IOCTLSOCKET OR - HAVE_IOCTLSOCKET_CASE OR - HAVE_O_NONBLOCK) +endif() if(RETSIGTYPE_TEST) set(RETSIGTYPE void) -else(RETSIGTYPE_TEST) +else() set(RETSIGTYPE int) -endif(RETSIGTYPE_TEST) +endif() if(CMAKE_COMPILER_IS_GNUCC AND APPLE) include(CheckCCompilerFlag) @@ -1185,13 +1182,13 @@ if(CMAKE_COMPILER_IS_GNUCC AND APPLE) get_source_file_property(MPRINTF_COMPILE_FLAGS mprintf.c COMPILE_FLAGS) if(MPRINTF_COMPILE_FLAGS) set(MPRINTF_COMPILE_FLAGS "${MPRINTF_COMPILE_FLAGS} -Wno-long-double") - else(MPRINTF_COMPILE_FLAGS) + else() set(MPRINTF_COMPILE_FLAGS "-Wno-long-double") - endif(MPRINTF_COMPILE_FLAGS) + endif() set_source_files_properties(mprintf.c PROPERTIES COMPILE_FLAGS ${MPRINTF_COMPILE_FLAGS}) - endif(HAVE_C_FLAG_Wno_long_double) -endif(CMAKE_COMPILER_IS_GNUCC AND APPLE) + endif() +endif() if(HAVE_SOCKLEN_T) set(CURL_HAVE_SOCKLEN_T 1) @@ -1226,19 +1223,25 @@ include(CMake/OtherTests.cmake) add_definitions(-DHAVE_CONFIG_H) -# For windows, all compilers used by cmake should support large files +# For Windows, all compilers used by CMake should support large files if(WIN32) set(USE_WIN32_LARGE_FILES ON) -endif(WIN32) + + # Use the manifest embedded in the Windows Resource + set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -DCURL_EMBED_MANIFEST") +endif() if(MSVC) + # Disable default manifest added by CMake + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") + add_definitions(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) if(CMAKE_C_FLAGS MATCHES "/W[0-4]") string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") - else(CMAKE_C_FLAGS MATCHES "/W[0-4]") + else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4") - endif(CMAKE_C_FLAGS MATCHES "/W[0-4]") -endif(MSVC) + endif() +endif() if(CURL_WERROR) if(MSVC_VERSION) @@ -1247,7 +1250,7 @@ if(CURL_WERROR) # this assumes clang or gcc style options set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") endif() -endif(CURL_WERROR) +endif() # Ugly (but functional) way to include "Makefile.inc" by transforming it (= regenerate it). function(TRANSFORM_MAKEFILE_INC INPUT_FILE OUTPUT_FILE) @@ -1266,9 +1269,9 @@ function(TRANSFORM_MAKEFILE_INC INPUT_FILE OUTPUT_FILE) endfunction() if(WIN32 AND NOT CYGWIN) - set(CURL_INSTALL_CMAKE_DIR CMake) + set(CURL_INSTALL_CMAKE_DIR CMake) else() - set(CURL_INSTALL_CMAKE_DIR lib/cmake/curl) + set(CURL_INSTALL_CMAKE_DIR lib/cmake/curl) endif() if(USE_MANUAL) @@ -1286,7 +1289,7 @@ endif() add_executable(LIBCURL curltest.c) target_link_libraries(LIBCURL cmcurl) -if(CMAKE_CURL_TEST_URL) +if(BUILD_TESTING AND CMAKE_CURL_TEST_URL) add_test(curl LIBCURL ${CMAKE_CURL_TEST_URL}) endif() @@ -1381,11 +1384,12 @@ set(CONFIGURE_OPTIONS "") # TODO when to set "-DCURL_STATICLIB" for CPPFLAG_CURL_STATICLIB? set(CPPFLAG_CURL_STATICLIB "") set(CURLVERSION "${CURL_VERSION}") -set(ENABLE_SHARED "yes") -if(CURL_STATICLIB) - set(ENABLE_STATIC "yes") -else() +if(BUILD_SHARED_LIBS) + set(ENABLE_SHARED "yes") set(ENABLE_STATIC "no") +else() + set(ENABLE_SHARED "no") + set(ENABLE_STATIC "yes") endif() set(exec_prefix "\${prefix}") set(includedir "\${prefix}/include") @@ -1409,6 +1413,9 @@ set(REQUIRE_LIB_DEPS "no") set(VERSIONNUM "${CURL_VERSION_NUM}") # Finally generate a "curl-config" matching this config +# Use: +# * ENABLE_SHARED +# * ENABLE_STATIC configure_file("${CURL_SOURCE_DIR}/curl-config.in" "${CURL_BINARY_DIR}/curl-config" @ONLY) install(FILES "${CURL_BINARY_DIR}/curl-config" @@ -1442,9 +1449,9 @@ write_basic_package_version_file( COMPATIBILITY SameMajorVersion ) -configure_file(CMake/curl-config.cmake +configure_file(CMake/curl-config.cmake.in "${PROJECT_BINARY_DIR}/curl-config.cmake" - COPYONLY + @ONLY ) install( diff --git a/Utilities/cmcurl/include/curl/curl.h b/Utilities/cmcurl/include/curl/curl.h index 10e5c49..d9955bd 100644 --- a/Utilities/cmcurl/include/curl/curl.h +++ b/Utilities/cmcurl/include/curl/curl.h @@ -691,6 +691,7 @@ typedef enum { * CURLAUTH_NTLM - HTTP NTLM authentication * CURLAUTH_DIGEST_IE - HTTP Digest authentication with IE flavour * CURLAUTH_NTLM_WB - HTTP NTLM authentication delegated to winbind helper + * CURLAUTH_BEARER - HTTP Bearer token authentication * CURLAUTH_ONLY - Use together with a single other type to force no * authentication or just that single type * CURLAUTH_ANY - All fine types set @@ -708,6 +709,7 @@ typedef enum { #define CURLAUTH_NTLM (((unsigned long)1)<<3) #define CURLAUTH_DIGEST_IE (((unsigned long)1)<<4) #define CURLAUTH_NTLM_WB (((unsigned long)1)<<5) +#define CURLAUTH_BEARER (((unsigned long)1)<<6) #define CURLAUTH_ONLY (((unsigned long)1)<<31) #define CURLAUTH_ANY (~CURLAUTH_DIGEST_IE) #define CURLAUTH_ANYSAFE (~(CURLAUTH_BASIC|CURLAUTH_DIGEST_IE)) @@ -1847,6 +1849,13 @@ typedef enum { /* shuffle addresses before use when DNS returns multiple */ CINIT(DNS_SHUFFLE_ADDRESSES, LONG, 275), + /* Specify which TLS 1.3 ciphers suites to use */ + CINIT(TLS13_CIPHERS, STRINGPOINT, 276), + CINIT(PROXY_TLS13_CIPHERS, STRINGPOINT, 277), + + /* Disallow specifying username/login in URL. */ + CINIT(DISALLOW_USERNAME_IN_URL, LONG, 278), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; @@ -2527,7 +2536,17 @@ typedef enum { CURLINFO_SCHEME = CURLINFO_STRING + 49, /* Fill in new entries below here! */ - CURLINFO_LASTONE = 49 + /* Preferably these would be defined conditionally based on the + sizeof curl_off_t being 64-bits */ + CURLINFO_TOTAL_TIME_T = CURLINFO_OFF_T + 50, + CURLINFO_NAMELOOKUP_TIME_T = CURLINFO_OFF_T + 51, + CURLINFO_CONNECT_TIME_T = CURLINFO_OFF_T + 52, + CURLINFO_PRETRANSFER_TIME_T = CURLINFO_OFF_T + 53, + CURLINFO_STARTTRANSFER_TIME_T = CURLINFO_OFF_T + 54, + CURLINFO_REDIRECT_TIME_T = CURLINFO_OFF_T + 55, + CURLINFO_APPCONNECT_TIME_T = CURLINFO_OFF_T + 56, + + CURLINFO_LASTONE = 56 } CURLINFO; /* CURLINFO_RESPONSE_CODE is the new name for the option previously known as @@ -2570,6 +2589,7 @@ typedef enum { CURL_LOCK_DATA_DNS, CURL_LOCK_DATA_SSL_SESSION, CURL_LOCK_DATA_CONNECT, + CURL_LOCK_DATA_PSL, CURL_LOCK_DATA_LAST } curl_lock_data; diff --git a/Utilities/cmcurl/include/curl/curlver.h b/Utilities/cmcurl/include/curl/curlver.h index 2d1505a..e266f18 100644 --- a/Utilities/cmcurl/include/curl/curlver.h +++ b/Utilities/cmcurl/include/curl/curlver.h @@ -30,13 +30,13 @@ /* This is the version number of the libcurl package from which this header file origins: */ -#define LIBCURL_VERSION "7.60.0" +#define LIBCURL_VERSION "7.61.1" /* The numeric version number is also available "in parts" by using these defines: */ #define LIBCURL_VERSION_MAJOR 7 -#define LIBCURL_VERSION_MINOR 60 -#define LIBCURL_VERSION_PATCH 0 +#define LIBCURL_VERSION_MINOR 61 +#define LIBCURL_VERSION_PATCH 1 /* This is the numeric version of the libcurl version number, meant for easier parsing and comparions by programs. The LIBCURL_VERSION_NUM define will @@ -57,7 +57,7 @@ CURL_VERSION_BITS() macro since curl's own configure script greps for it and needs it to contain the full number. */ -#define LIBCURL_VERSION_NUM 0x073C00 +#define LIBCURL_VERSION_NUM 0x073D01 /* * This is the date and time when the full source package was created. The diff --git a/Utilities/cmcurl/include/curl/system.h b/Utilities/cmcurl/include/curl/system.h index eac4cfe..a54fd58 100644 --- a/Utilities/cmcurl/include/curl/system.h +++ b/Utilities/cmcurl/include/curl/system.h @@ -320,6 +320,24 @@ # define CURL_PULL_SYS_TYPES_H 1 # define CURL_PULL_SYS_SOCKET_H 1 +#elif defined(__xlc__) /* IBM xlc compiler */ +# if !defined(_LP64) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 + /* ===================================== */ /* KEEP MSVC THE PENULTIMATE ENTRY */ /* ===================================== */ @@ -344,7 +362,7 @@ /* KEEP GENERIC GCC THE LAST ENTRY */ /* ===================================== */ -#elif defined(__GNUC__) +#elif defined(__GNUC__) && !defined(_SCO_DS) # if !defined(__LP64__) && \ (defined(__ILP32__) || defined(__i386__) || defined(__hppa__) || \ defined(__ppc__) || defined(__powerpc__) || defined(__arm__) || \ diff --git a/Utilities/cmcurl/include/curl/typecheck-gcc.h b/Utilities/cmcurl/include/curl/typecheck-gcc.h index 3a0f253..a6f6386 100644 --- a/Utilities/cmcurl/include/curl/typecheck-gcc.h +++ b/Utilities/cmcurl/include/curl/typecheck-gcc.h @@ -428,7 +428,7 @@ _CURL_WARNING(_curl_easy_getinfo_err_curl_off_t, * == or whatsoever. */ -/* XXX: should evaluate to true iff expr is a pointer */ +/* XXX: should evaluate to true if expr is a pointer */ #define _curl_is_any_ptr(expr) \ (sizeof(expr) == sizeof(void *)) diff --git a/Utilities/cmcurl/lib/CMakeLists.txt b/Utilities/cmcurl/lib/CMakeLists.txt index 9f19daa..3d52105 100644 --- a/Utilities/cmcurl/lib/CMakeLists.txt +++ b/Utilities/cmcurl/lib/CMakeLists.txt @@ -1,5 +1,13 @@ set(LIB_NAME cmcurl) +if(BUILD_SHARED_LIBS) + set(CURL_STATICLIB NO) +else() + set(CURL_STATICLIB YES) +endif() + +# Use: +# * CURL_STATICLIB configure_file(curl_config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/curl_config.h) @@ -59,14 +67,6 @@ if(USE_ARES) include_directories(${CARES_INCLUDE_DIR}) endif() -if(CURL_STATICLIB) - # Static lib - set(CURL_USER_DEFINED_DYNAMIC_OR_STATIC STATIC) -else() - # DLL / so dynamic lib - set(CURL_USER_DEFINED_DYNAMIC_OR_STATIC SHARED) -endif() - # For windows we want to install OPENSSL_LIBRARIES dlls # and also copy them into the build tree so that testing # can find them. @@ -93,16 +93,19 @@ endif() add_library( ${LIB_NAME} - ${CURL_USER_DEFINED_DYNAMIC_OR_STATIC} ${HHEADERS} ${CSOURCES} ${CMAKE_CURL_SSL_DLLS} ) +if(NOT BUILD_SHARED_LIBS) + set_target_properties(${LIB_NAME} PROPERTIES INTERFACE_COMPILE_DEFINITIONS CURL_STATICLIB) +endif() + target_link_libraries(${LIB_NAME} ${CURL_LIBS}) if(0) # This code not needed for building within CMake. if(WIN32) - add_definitions( -D_USRDLL ) + add_definitions(-D_USRDLL) endif() endif() @@ -119,21 +122,14 @@ set_target_properties(${LIB_NAME} PROPERTIES PREFIX "") set_target_properties(${LIB_NAME} PROPERTIES IMPORT_PREFIX "") if(WIN32) - if(NOT CURL_STATICLIB) + if(BUILD_SHARED_LIBS) # Add "_imp" as a suffix before the extension to avoid conflicting with the statically linked "libcurl.lib" set_target_properties(${LIB_NAME} PROPERTIES IMPORT_SUFFIX "_imp.lib") - - set_target_properties (${LIB_NAME} PROPERTIES - DEBUG_POSTFIX "-d" - # Note: no postfix for release variants, let user choose what style of release he wants - # MINSIZEREL_POSTFIX "-z" - # RELWITHDEBINFO_POSTFIX "-g" - ) endif() endif() target_include_directories(${LIB_NAME} INTERFACE - $<INSTALL_INTERFACE:include>) + $<INSTALL_INTERFACE:include>) install(TARGETS ${LIB_NAME} EXPORT libcurl-target diff --git a/Utilities/cmcurl/lib/Makefile.inc b/Utilities/cmcurl/lib/Makefile.inc index 61c2341..76ca6d0 100644 --- a/Utilities/cmcurl/lib/Makefile.inc +++ b/Utilities/cmcurl/lib/Makefile.inc @@ -54,7 +54,7 @@ LIB_CFILES = file.c timeval.c base64.c hostip.c progress.c formdata.c \ http_ntlm.c curl_ntlm_wb.c curl_ntlm_core.c curl_sasl.c rand.c \ curl_multibyte.c hostcheck.c conncache.c pipeline.c dotdot.c \ x509asn1.c http2.c smb.c curl_endian.c curl_des.c system_win32.c \ - mime.c sha256.c setopt.c curl_path.c curl_ctype.c curl_range.c + mime.c sha256.c setopt.c curl_path.c curl_ctype.c curl_range.c psl.c LIB_HFILES = arpa_telnet.h netrc.h file.h timeval.h hostip.h progress.h \ formdata.h cookie.h http.h sendf.h ftp.h url.h dict.h if2ip.h \ @@ -74,7 +74,7 @@ LIB_HFILES = arpa_telnet.h netrc.h file.h timeval.h hostip.h progress.h \ curl_setup_once.h multihandle.h setup-vms.h pipeline.h dotdot.h \ x509asn1.h http2.h sigpipe.h smb.h curl_endian.h curl_des.h \ curl_printf.h system_win32.h rand.h mime.h curl_sha256.h setopt.h \ - curl_path.h curl_ctype.h curl_range.h + curl_path.h curl_ctype.h curl_range.h psl.h LIB_RCFILES = libcurl.rc diff --git a/Utilities/cmcurl/lib/asyn-ares.c b/Utilities/cmcurl/lib/asyn-ares.c index aa581a4..5cfb260 100644 --- a/Utilities/cmcurl/lib/asyn-ares.c +++ b/Utilities/cmcurl/lib/asyn-ares.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -312,22 +312,25 @@ CURLcode Curl_resolver_is_resolved(struct connectdata *conn, conn->async.os_specific; CURLcode result = CURLE_OK; - *dns = NULL; + if(dns) + *dns = NULL; waitperform(conn, 0); if(res && !res->num_pending) { - (void)Curl_addrinfo_callback(conn, res->last_status, res->temp_ai); - /* temp_ai ownership is moved to the connection, so we need not free-up - them */ - res->temp_ai = NULL; + if(dns) { + (void)Curl_addrinfo_callback(conn, res->last_status, res->temp_ai); + /* temp_ai ownership is moved to the connection, so we need not free-up + them */ + res->temp_ai = NULL; + } if(!conn->async.dns) { failf(data, "Could not resolve: %s (%s)", conn->async.hostname, ares_strerror(conn->async.status)); result = conn->bits.proxy?CURLE_COULDNT_RESOLVE_PROXY: CURLE_COULDNT_RESOLVE_HOST; } - else + else if(dns) *dns = conn->async.dns; destroy_async_data(&conn->async); @@ -390,7 +393,7 @@ CURLcode Curl_resolver_wait_resolv(struct connectdata *conn, timeout_ms = 1000; waitperform(conn, timeout_ms); - result = Curl_resolver_is_resolved(conn, &temp_entry); + result = Curl_resolver_is_resolved(conn, entry?&temp_entry:NULL); if(result || conn->async.done) break; @@ -472,17 +475,19 @@ static void query_completed_cb(void *arg, /* (struct connectdata *) */ return; res = (struct ResolverResults *)conn->async.os_specific; - res->num_pending--; + if(res) { + res->num_pending--; - if(CURL_ASYNC_SUCCESS == status) { - Curl_addrinfo *ai = Curl_he2ai(hostent, conn->async.port); - if(ai) { - compound_results(res, ai); + if(CURL_ASYNC_SUCCESS == status) { + Curl_addrinfo *ai = Curl_he2ai(hostent, conn->async.port); + if(ai) { + compound_results(res, ai); + } } + /* A successful result overwrites any previous error */ + if(res->last_status != ARES_SUCCESS) + res->last_status = status; } - /* A successful result overwrites any previous error */ - if(res->last_status != ARES_SUCCESS) - res->last_status = status; } /* diff --git a/Utilities/cmcurl/lib/asyn-thread.c b/Utilities/cmcurl/lib/asyn-thread.c index b11fab2..2a59294 100644 --- a/Utilities/cmcurl/lib/asyn-thread.c +++ b/Utilities/cmcurl/lib/asyn-thread.c @@ -182,8 +182,6 @@ static struct thread_sync_data *conn_thread_sync_data(struct connectdata *conn) return &(((struct thread_data *)conn->async.os_specific)->tsd); } -#define CONN_THREAD_SYNC_DATA(conn) &(((conn)->async.os_specific)->tsd); - /* Destroy resolver thread synchronization data */ static void destroy_thread_sync_data(struct thread_sync_data * tsd) @@ -481,8 +479,10 @@ CURLcode Curl_resolver_wait_resolv(struct connectdata *conn, DEBUGASSERT(conn && td); /* wait for the thread to resolve the name */ - if(Curl_thread_join(&td->thread_hnd)) - result = getaddrinfo_complete(conn); + if(Curl_thread_join(&td->thread_hnd)) { + if(entry) + result = getaddrinfo_complete(conn); + } else DEBUGASSERT(0); @@ -572,10 +572,10 @@ int Curl_resolver_getsock(struct connectdata *conn, (void)socks; (void)numsocks; ms = Curl_timediff(Curl_now(), reslv->start); - if(ms < 10) - milli = ms/3; + if(ms < 3) + milli = 0; else if(ms <= 50) - milli = 10; + milli = ms/3; else if(ms <= 250) milli = 50; else diff --git a/Utilities/cmcurl/lib/base64.c b/Utilities/cmcurl/lib/base64.c index 204a227..6370e4c 100644 --- a/Utilities/cmcurl/lib/base64.c +++ b/Utilities/cmcurl/lib/base64.c @@ -49,13 +49,12 @@ static size_t decodeQuantum(unsigned char *dest, const char *src) unsigned long i, x = 0; for(i = 0, s = src; i < 4; i++, s++) { - unsigned long v = 0; - if(*s == '=') { x = (x << 6); padding++; } else { + unsigned long v = 0; p = base64; while(*p && (*p != *s)) { diff --git a/Utilities/cmcurl/lib/conncache.c b/Utilities/cmcurl/lib/conncache.c index b8f5444..6fbf3b1 100644 --- a/Utilities/cmcurl/lib/conncache.c +++ b/Utilities/cmcurl/lib/conncache.c @@ -6,7 +6,7 @@ * \___|\___/|_| \_\_____| * * Copyright (C) 2012 - 2016, Linus Nielsen Feltzing, <linus@haxx.se> - * Copyright (C) 2012 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 2012 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -63,10 +63,9 @@ static void conn_llist_dtor(void *user, void *element) { - struct connectdata *data = element; + struct connectdata *conn = element; (void)user; - - data->bundle = NULL; + conn->bundle = NULL; } static CURLcode bundle_create(struct Curl_easy *data, @@ -96,14 +95,13 @@ static void bundle_destroy(struct connectbundle *cb_ptr) } /* Add a connection to a bundle */ -static CURLcode bundle_add_conn(struct connectbundle *cb_ptr, - struct connectdata *conn) +static void bundle_add_conn(struct connectbundle *cb_ptr, + struct connectdata *conn) { Curl_llist_insert_next(&cb_ptr->conn_list, cb_ptr->conn_list.tail, conn, &conn->bundle_node); conn->bundle = cb_ptr; cb_ptr->num_connections++; - return CURLE_OK; } /* Remove a connection from a bundle */ @@ -263,7 +261,7 @@ static void conncache_remove_bundle(struct conncache *connc, CURLcode Curl_conncache_add_conn(struct conncache *connc, struct connectdata *conn) { - CURLcode result; + CURLcode result = CURLE_OK; struct connectbundle *bundle; struct connectbundle *new_bundle = NULL; struct Curl_easy *data = conn->data; @@ -290,19 +288,13 @@ CURLcode Curl_conncache_add_conn(struct conncache *connc, bundle = new_bundle; } - result = bundle_add_conn(bundle, conn); - if(result) { - if(new_bundle) - conncache_remove_bundle(data->state.conn_cache, new_bundle); - goto unlock; - } - + bundle_add_conn(bundle, conn); conn->connection_id = connc->next_connection_id++; connc->num_conn++; DEBUGF(infof(conn->data, "Added connection %ld. " - "The cache now contains %" CURL_FORMAT_CURL_OFF_TU " members\n", - conn->connection_id, (curl_off_t) connc->num_conn)); + "The cache now contains %zu members\n", + conn->connection_id, connc->num_conn)); unlock: CONN_UNLOCK(data); @@ -320,7 +312,7 @@ void Curl_conncache_remove_conn(struct connectdata *conn, bool lock) due to a failed connection attempt, before being added to a bundle */ if(bundle) { if(lock) { - CONN_LOCK(conn->data); + CONN_LOCK(data); } bundle_remove_conn(bundle, conn); if(bundle->num_connections == 0) @@ -328,12 +320,11 @@ void Curl_conncache_remove_conn(struct connectdata *conn, bool lock) conn->bundle = NULL; /* removed from it */ if(connc) { connc->num_conn--; - DEBUGF(infof(conn->data, "The cache now contains %" - CURL_FORMAT_CURL_OFF_TU " members\n", - (curl_off_t) connc->num_conn)); + DEBUGF(infof(data, "The cache now contains %zu members\n", + connc->num_conn)); } if(lock) { - CONN_UNLOCK(conn->data); + CONN_UNLOCK(data); } } } @@ -441,18 +432,11 @@ bool Curl_conncache_return_conn(struct connectdata *conn) infof(data, "Connection cache is full, closing the oldest one.\n"); conn_candidate = Curl_conncache_extract_oldest(data); - if(conn_candidate) { - /* Set the connection's owner correctly */ - conn_candidate->data = data; - /* the winner gets the honour of being disconnected */ - (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE); + (void)Curl_disconnect(data, conn_candidate, /* dead_connection */ FALSE); } } - CONN_LOCK(data); - conn->inuse = FALSE; /* Mark the connection unused */ - CONN_UNLOCK(data); return (conn_candidate == conn) ? FALSE : TRUE; @@ -486,7 +470,7 @@ Curl_conncache_extract_bundle(struct Curl_easy *data, while(curr) { conn = curr->ptr; - if(!conn->inuse) { + if(!CONN_INUSE(conn)) { /* Set higher score for the age passed since the connection was used */ score = Curl_timediff(now, conn->now); @@ -501,9 +485,9 @@ Curl_conncache_extract_bundle(struct Curl_easy *data, /* remove it to prevent another thread from nicking it */ bundle_remove_conn(bundle, conn_candidate); data->state.conn_cache->num_conn--; - DEBUGF(infof(data, "The cache now contains %" - CURL_FORMAT_CURL_OFF_TU " members\n", - (curl_off_t) data->state.conn_cache->num_conn)); + DEBUGF(infof(data, "The cache now contains %zu members\n", + data->state.conn_cache->num_conn)); + conn_candidate->data = data; /* associate! */ } return conn_candidate; @@ -544,7 +528,7 @@ Curl_conncache_extract_oldest(struct Curl_easy *data) while(curr) { conn = curr->ptr; - if(!conn->inuse) { + if(!CONN_INUSE(conn)) { /* Set higher score for the age passed since the connection was used */ score = Curl_timediff(now, conn->now); @@ -563,9 +547,9 @@ Curl_conncache_extract_oldest(struct Curl_easy *data) /* remove it to prevent another thread from nicking it */ bundle_remove_conn(bundle_candidate, conn_candidate); connc->num_conn--; - DEBUGF(infof(data, "The cache now contains %" - CURL_FORMAT_CURL_OFF_TU " members\n", - (curl_off_t) connc->num_conn)); + DEBUGF(infof(data, "The cache now contains %zu members\n", + connc->num_conn)); + conn_candidate->data = data; /* associate! */ } CONN_UNLOCK(data); @@ -586,7 +570,7 @@ void Curl_conncache_close_all_connections(struct conncache *connc) pointer */ /* This will remove the connection from the cache */ connclose(conn, "kill all"); - (void)Curl_disconnect(conn, FALSE); + (void)Curl_disconnect(connc->closure_handle, conn, FALSE); sigpipe_restore(&pipe_st); conn = Curl_conncache_find_first_connection(connc); diff --git a/Utilities/cmcurl/lib/conncache.h b/Utilities/cmcurl/lib/conncache.h index d8ad80f..eedd7a8 100644 --- a/Utilities/cmcurl/lib/conncache.h +++ b/Utilities/cmcurl/lib/conncache.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 2015 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 2015 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * Copyright (C) 2012 - 2014, Linus Nielsen Feltzing, <linus@haxx.se> * * This software is licensed as described in the file COPYING, which @@ -63,7 +63,7 @@ size_t Curl_conncache_bundle_size(struct connectdata *conn); bool Curl_conncache_return_conn(struct connectdata *conn); CURLcode Curl_conncache_add_conn(struct conncache *connc, - struct connectdata *conn); + struct connectdata *conn) WARN_UNUSED_RESULT; void Curl_conncache_remove_conn(struct connectdata *conn, bool lock); bool Curl_conncache_foreach(struct Curl_easy *data, diff --git a/Utilities/cmcurl/lib/connect.c b/Utilities/cmcurl/lib/connect.c index 1a27ae1..41f2202 100644 --- a/Utilities/cmcurl/lib/connect.c +++ b/Utilities/cmcurl/lib/connect.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -1237,8 +1237,6 @@ static int conn_is_conn(struct connectdata *conn, void *param) curl_socket_t Curl_getconnectinfo(struct Curl_easy *data, struct connectdata **connp) { - curl_socket_t sockfd; - DEBUGASSERT(data); /* this works for an easy handle: @@ -1261,15 +1259,15 @@ curl_socket_t Curl_getconnectinfo(struct Curl_easy *data, return CURL_SOCKET_BAD; } - if(connp) + if(connp) { /* only store this if the caller cares for it */ *connp = c; - sockfd = c->sock[FIRSTSOCKET]; + c->data = data; + } + return c->sock[FIRSTSOCKET]; } else return CURL_SOCKET_BAD; - - return sockfd; } /* diff --git a/Utilities/cmcurl/lib/content_encoding.c b/Utilities/cmcurl/lib/content_encoding.c index 7c979ef..6d47537 100644 --- a/Utilities/cmcurl/lib/content_encoding.c +++ b/Utilities/cmcurl/lib/content_encoding.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -71,13 +71,13 @@ #define RESERVED 0xE0 /* bits 5..7: reserved */ typedef enum { - ZLIB_UNINIT, /* uninitialized */ - ZLIB_INIT, /* initialized */ - ZLIB_INFLATING, /* Inflating started. */ - ZLIB_GZIP_HEADER, /* reading gzip header */ - ZLIB_GZIP_TRAILER, /* reading gzip trailer */ - ZLIB_GZIP_INFLATING, /* inflating gzip stream */ - ZLIB_INIT_GZIP /* initialized in transparent gzip mode */ + ZLIB_UNINIT, /* uninitialized */ + ZLIB_INIT, /* initialized */ + ZLIB_INFLATING, /* inflating started. */ + ZLIB_EXTERNAL_TRAILER, /* reading external trailer */ + ZLIB_GZIP_HEADER, /* reading gzip header */ + ZLIB_GZIP_INFLATING, /* inflating gzip stream */ + ZLIB_INIT_GZIP /* initialized in transparent gzip mode */ } zlibInitState; /* Writer parameters. */ @@ -150,8 +150,8 @@ static CURLcode process_trailer(struct connectdata *conn, zlib_params *zp) if(result || !zp->trailerlen) result = exit_zlib(conn, z, &zp->zlib_init, result); else { - /* Only occurs for gzip with zlib < 1.2.0.4. */ - zp->zlib_init = ZLIB_GZIP_TRAILER; + /* Only occurs for gzip with zlib < 1.2.0.4 or raw deflate. */ + zp->zlib_init = ZLIB_EXTERNAL_TRAILER; } return result; } @@ -163,7 +163,6 @@ static CURLcode inflate_stream(struct connectdata *conn, z_stream *z = &zp->z; /* zlib state structure */ uInt nread = z->avail_in; Bytef *orig_in = z->next_in; - int status; /* zlib status */ bool done = FALSE; CURLcode result = CURLE_OK; /* Curl_client_write status */ char *decomp; /* Put the decompressed data here. */ @@ -184,13 +183,20 @@ static CURLcode inflate_stream(struct connectdata *conn, /* because the buffer size is fixed, iteratively decompress and transfer to the client via downstream_write function. */ while(!done) { + int status; /* zlib status */ done = TRUE; /* (re)set buffer for decompressed output for every iteration */ z->next_out = (Bytef *) decomp; z->avail_out = DSIZ; +#ifdef Z_BLOCK + /* Z_BLOCK is only available in zlib ver. >= 1.2.0.5 */ status = inflate(z, Z_BLOCK); +#else + /* fallback for zlib ver. < 1.2.0.5 */ + status = inflate(z, Z_SYNC_FLUSH); +#endif /* Flush output data if some. */ if(z->avail_out != DSIZ) { @@ -227,6 +233,7 @@ static CURLcode inflate_stream(struct connectdata *conn, z->next_in = orig_in; z->avail_in = nread; zp->zlib_init = ZLIB_INFLATING; + zp->trailerlen = 4; /* Tolerate up to 4 unknown trailer bytes. */ done = FALSE; break; } @@ -281,6 +288,9 @@ static CURLcode deflate_unencode_write(struct connectdata *conn, z->next_in = (Bytef *) buf; z->avail_in = (uInt) nbytes; + if(zp->zlib_init == ZLIB_EXTERNAL_TRAILER) + return process_trailer(conn, zp); + /* Now uncompress the data */ return inflate_stream(conn, writer, ZLIB_INFLATING); } @@ -526,7 +536,7 @@ static CURLcode gzip_unencode_write(struct connectdata *conn, } break; - case ZLIB_GZIP_TRAILER: + case ZLIB_EXTERNAL_TRAILER: z->next_in = (Bytef *) buf; z->avail_in = (uInt) nbytes; return process_trailer(conn, zp); @@ -755,7 +765,6 @@ char *Curl_all_content_encodings(void) const content_encoding * const *cep; const content_encoding *ce; char *ace; - char *p; for(cep = encodings; *cep; cep++) { ce = *cep; @@ -768,7 +777,7 @@ char *Curl_all_content_encodings(void) ace = malloc(len); if(ace) { - p = ace; + char *p = ace; for(cep = encodings; *cep; cep++) { ce = *cep; if(!strcasecompare(ce->name, CONTENT_ENCODING_DEFAULT)) { @@ -915,10 +924,9 @@ void Curl_unencode_cleanup(struct connectdata *conn) static const content_encoding *find_encoding(const char *name, size_t len) { const content_encoding * const *cep; - const content_encoding *ce; for(cep = encodings; *cep; cep++) { - ce = *cep; + const content_encoding *ce = *cep; if((strncasecompare(name, ce->name, len) && !ce->name[len]) || (ce->alias && strncasecompare(name, ce->alias, len) && !ce->alias[len])) return ce; diff --git a/Utilities/cmcurl/lib/cookie.c b/Utilities/cmcurl/lib/cookie.c index 2c028b3..da49c2a 100644 --- a/Utilities/cmcurl/lib/cookie.c +++ b/Utilities/cmcurl/lib/cookie.c @@ -40,7 +40,7 @@ struct Cookie *Curl_cookie_add(struct Curl_easy *data, received from a server. The function need to replace previously stored lines that this new - line superceeds. + line supersedes. It may remove lines that are expired. @@ -84,12 +84,9 @@ Example set of cookies: #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES) -#ifdef USE_LIBPSL -# include <libpsl.h> -#endif - #include "urldata.h" #include "cookie.h" +#include "psl.h" #include "strtok.h" #include "sendf.h" #include "slist.h" @@ -253,9 +250,9 @@ static const char *get_top_domain(const char * const domain, size_t *outlen) len = strlen(domain); last = memrchr(domain, '.', len); if(last) { - first = memrchr(domain, '.', (size_t) (last - domain)); + first = memrchr(domain, '.', (last - domain)); if(first) - len -= (size_t) (++first - domain); + len -= (++first - domain); } if(outlen) @@ -379,13 +376,13 @@ static void strstore(char **str, const char *newstr) */ static void remove_expired(struct CookieInfo *cookies) { - struct Cookie *co, *nx, *pv; + struct Cookie *co, *nx; curl_off_t now = (curl_off_t)time(NULL); unsigned int i; for(i = 0; i < COOKIE_HASH_SIZE; i++) { + struct Cookie *pv = NULL; co = cookies->cookies[i]; - pv = NULL; while(co) { nx = co->next; if(co->expires && co->expires < now) { @@ -406,6 +403,12 @@ static void remove_expired(struct CookieInfo *cookies) } } +/* Make sure domain contains a dot or is localhost. */ +static bool bad_domain(const char *domain) +{ + return !strchr(domain, '.') && !strcasecompare(domain, "localhost"); +} + /**************************************************************************** * * Curl_cookie_add() @@ -442,10 +445,6 @@ Curl_cookie_add(struct Curl_easy *data, bool badcookie = FALSE; /* cookies are good by default. mmmmm yummy */ size_t myhash; -#ifdef USE_LIBPSL - const psl_ctx_t *psl; -#endif - #ifdef CURL_DISABLE_VERBOSE_STRINGS (void)data; #endif @@ -497,7 +496,7 @@ Curl_cookie_add(struct Curl_easy *data, name + contents. Chrome and Firefox support 4095 or 4096 bytes combo. */ freecookie(co); - infof(data, "oversized cookie dropped, name/val %d + %d bytes\n", + infof(data, "oversized cookie dropped, name/val %zu + %zu bytes\n", nlen, len); return NULL; } @@ -585,13 +584,8 @@ Curl_cookie_add(struct Curl_easy *data, * TLD or otherwise "protected" suffix. To reduce risk, we require a * dot OR the exact host name being "localhost". */ - { - const char *dotp; - /* check for more dots */ - dotp = strchr(whatptr, '.'); - if(!dotp && !strcasecompare("localhost", whatptr)) - domain = ":"; - } + if(bad_domain(whatptr)) + domain = ":"; #endif is_ip = isip(domain ? domain : whatptr); @@ -723,9 +717,9 @@ Curl_cookie_add(struct Curl_easy *data, if(!queryp) endslash = strrchr(path, '/'); else - endslash = memrchr(path, '/', (size_t)(queryp - path)); + endslash = memrchr(path, '/', (queryp - path)); if(endslash) { - size_t pathlen = (size_t)(endslash-path + 1); /* include end slash */ + size_t pathlen = (endslash-path + 1); /* include end slash */ co->path = malloc(pathlen + 1); /* one extra for the zero byte */ if(co->path) { memcpy(co->path, path, pathlen); @@ -880,9 +874,10 @@ Curl_cookie_add(struct Curl_easy *data, } co->livecookie = c->running; + co->creationtime = ++c->lastct; /* now, we have parsed the incoming line, we must now check if this - superceeds an already existing cookie, which it may if the previous have + supersedes an already existing cookie, which it may if the previous have the same domain and path as this */ /* at first, remove expired cookies */ @@ -890,14 +885,21 @@ Curl_cookie_add(struct Curl_easy *data, remove_expired(c); #ifdef USE_LIBPSL - /* Check if the domain is a Public Suffix and if yes, ignore the cookie. - This needs a libpsl compiled with builtin data. */ + /* Check if the domain is a Public Suffix and if yes, ignore the cookie. */ if(domain && co->domain && !isip(co->domain)) { - psl = psl_builtin(); - if(psl && !psl_is_cookie_domain_acceptable(psl, domain, co->domain)) { - infof(data, - "cookie '%s' dropped, domain '%s' must not set cookies for '%s'\n", - co->name, domain, co->domain); + const psl_ctx_t *psl = Curl_psl_use(data); + int acceptable; + + if(psl) { + acceptable = psl_is_cookie_domain_acceptable(psl, domain, co->domain); + Curl_psl_release(data); + } + else + acceptable = !bad_domain(domain); + + if(!acceptable) { + infof(data, "cookie '%s' dropped, domain '%s' must not " + "set cookies for '%s'\n", co->name, domain, co->domain); freecookie(co); return NULL; } @@ -951,6 +953,9 @@ Curl_cookie_add(struct Curl_easy *data, if(replace_old) { co->next = clist->next; /* get the next-pointer first */ + /* when replacing, creationtime is kept from old */ + co->creationtime = clist->creationtime; + /* then free all the old pointers */ free(clist->name); free(clist->value); @@ -1140,12 +1145,24 @@ static int cookie_sort(const void *p1, const void *p2) if(l1 != l2) return (l2 > l1) ? 1 : -1 ; /* avoid size_t <=> int conversions */ - /* 3 - compare cookie names */ - if(c1->name && c2->name) - return strcmp(c1->name, c2->name); + /* 3 - compare cookie name lengths */ + l1 = c1->name ? strlen(c1->name) : 0; + l2 = c2->name ? strlen(c2->name) : 0; - /* sorry, can't be more deterministic */ - return 0; + if(l1 != l2) + return (l2 > l1) ? 1 : -1; + + /* 4 - compare cookie creation time */ + return (c2->creationtime > c1->creationtime) ? 1 : -1; +} + +/* sort cookies only according to creation time */ +static int cookie_sort_ct(const void *p1, const void *p2) +{ + struct Cookie *c1 = *(struct Cookie **)p1; + struct Cookie *c2 = *(struct Cookie **)p2; + + return (c2->creationtime > c1->creationtime) ? 1 : -1; } #define CLONE(field) \ @@ -1174,6 +1191,7 @@ static struct Cookie *dup_cookie(struct Cookie *src) d->secure = src->secure; d->livecookie = src->livecookie; d->httponly = src->httponly; + d->creationtime = src->creationtime; } return d; @@ -1384,9 +1402,8 @@ void Curl_cookie_clearsess(struct CookieInfo *cookies) ****************************************************************************/ void Curl_cookie_cleanup(struct CookieInfo *c) { - unsigned int i; - if(c) { + unsigned int i; free(c->filename); for(i = 0; i < COOKIE_HASH_SIZE; i++) Curl_cookie_freelist(c->cookies[i]); @@ -1439,6 +1456,8 @@ static int cookie_output(struct CookieInfo *c, const char *dumphere) bool use_stdout = FALSE; char *format_ptr; unsigned int i; + unsigned int j; + struct Cookie **array; if((NULL == c) || (0 == c->numcookies)) /* If there are no known cookies, we don't write or even create any @@ -1452,6 +1471,10 @@ static int cookie_output(struct CookieInfo *c, const char *dumphere) if(0 == c->numcookies) return 0; + array = malloc(sizeof(struct Cookie *) * c->numcookies); + if(!array) + return 1; + if(!strcmp("-", dumphere)) { /* use stdout */ out = stdout; @@ -1459,8 +1482,10 @@ static int cookie_output(struct CookieInfo *c, const char *dumphere) } else { out = fopen(dumphere, FOPEN_WRITETEXT); - if(!out) + if(!out) { + free(array); return 1; /* failure */ + } } fputs("# Netscape HTTP Cookie File\n" @@ -1468,22 +1493,33 @@ static int cookie_output(struct CookieInfo *c, const char *dumphere) "# This file was generated by libcurl! Edit at your own risk.\n\n", out); + j = 0; for(i = 0; i < COOKIE_HASH_SIZE; i++) { for(co = c->cookies[i]; co; co = co->next) { if(!co->domain) continue; - format_ptr = get_netscape_format(co); - if(format_ptr == NULL) { - fprintf(out, "#\n# Fatal libcurl error\n"); - if(!use_stdout) - fclose(out); - return 1; + array[j++] = co; + } + } + + qsort(array, c->numcookies, sizeof(struct Cookie *), cookie_sort_ct); + + for(i = 0; i < j; i++) { + format_ptr = get_netscape_format(array[i]); + if(format_ptr == NULL) { + fprintf(out, "#\n# Fatal libcurl error\n"); + if(!use_stdout) { + free(array); + fclose(out); } - fprintf(out, "%s\n", format_ptr); - free(format_ptr); + return 1; } + fprintf(out, "%s\n", format_ptr); + free(format_ptr); } + free(array); + if(!use_stdout) fclose(out); diff --git a/Utilities/cmcurl/lib/cookie.h b/Utilities/cmcurl/lib/cookie.h index 79b5928..a9f90ca 100644 --- a/Utilities/cmcurl/lib/cookie.h +++ b/Utilities/cmcurl/lib/cookie.h @@ -34,7 +34,7 @@ struct Cookie { char *domain; /* domain = <this> */ curl_off_t expires; /* expires = <this> */ char *expirestr; /* the plain text version */ - bool tailmatch; /* weather we do tail-matchning of the domain name */ + bool tailmatch; /* whether we do tail-matching of the domain name */ /* RFC 2109 keywords. Version=1 means 2109-compliant cookie sending */ char *version; /* Version = <value> */ @@ -43,6 +43,7 @@ struct Cookie { bool secure; /* whether the 'secure' keyword was used */ bool livecookie; /* updated from a server, not a stored file */ bool httponly; /* true if the httponly directive is present */ + int creationtime; /* time when the cookie was written */ }; #define COOKIE_HASH_SIZE 256 @@ -55,6 +56,7 @@ struct CookieInfo { bool running; /* state info, for cookie adding information */ long numcookies; /* number of cookies in the "jar" */ bool newsession; /* new session, discard session cookies on load */ + int lastct; /* last creation-time used in the jar */ }; /* This is the maximum line length we accept for a cookie line. RFC 2109 diff --git a/Utilities/cmcurl/lib/curl_addrinfo.c b/Utilities/cmcurl/lib/curl_addrinfo.c index 55d5a39..fd49679 100644 --- a/Utilities/cmcurl/lib/curl_addrinfo.c +++ b/Utilities/cmcurl/lib/curl_addrinfo.c @@ -536,7 +536,8 @@ Curl_addrinfo *Curl_unix2addr(const char *path, bool *longpath, bool abstract) } #endif -#if defined(CURLDEBUG) && defined(HAVE_FREEADDRINFO) +#if defined(CURLDEBUG) && defined(HAVE_GETADDRINFO) && \ + defined(HAVE_FREEADDRINFO) /* * curl_dofreeaddrinfo() * diff --git a/Utilities/cmcurl/lib/curl_config.h.cmake b/Utilities/cmcurl/lib/curl_config.h.cmake index 24d1ca0..f67188e 100644 --- a/Utilities/cmcurl/lib/curl_config.h.cmake +++ b/Utilities/cmcurl/lib/curl_config.h.cmake @@ -226,6 +226,9 @@ /* Define to 1 if you have the `getpwuid' function. */ #cmakedefine HAVE_GETPWUID 1 +/* Define to 1 if you have the `getpwuid_r' function. */ +#cmakedefine HAVE_GETPWUID_R 1 + /* Define to 1 if you have the `getrlimit' function. */ #cmakedefine HAVE_GETRLIMIT 1 @@ -876,6 +879,12 @@ /* The size of `long', as computed by sizeof. */ @SIZEOF_LONG_CODE@ +/* The size of `long long', as computed by sizeof. */ +@SIZEOF_LONG_LONG_CODE@ + +/* The size of `__int64', as computed by sizeof. */ +@SIZEOF___INT64_CODE@ + /* The size of `off_t', as computed by sizeof. */ @SIZEOF_OFF_T_CODE@ @@ -951,6 +960,9 @@ /* to enable Windows SSL */ #cmakedefine USE_SCHANNEL 1 +/* enable multiple SSL backends */ +#cmakedefine CURL_WITH_MULTI_SSL 1 + /* Define to 1 if using yaSSL in OpenSSL compatibility mode. */ #cmakedefine USE_YASSLEMUL 1 diff --git a/Utilities/cmcurl/lib/curl_fnmatch.c b/Utilities/cmcurl/lib/curl_fnmatch.c index 0179a4f..fbfd85c 100644 --- a/Utilities/cmcurl/lib/curl_fnmatch.c +++ b/Utilities/cmcurl/lib/curl_fnmatch.c @@ -30,6 +30,17 @@ /* The last #include file should be: */ #include "memdebug.h" +#ifndef HAVE_FNMATCH + +/* + * TODO: + * + * Make this function match POSIX. Test 1307 includes a set of test patterns + * that returns different results with a POSIX fnmatch() than with this + * implementation and this is considered a bug where POSIX is the guiding + * light. + */ + #define CURLFNM_CHARSET_LEN (sizeof(char) * 256) #define CURLFNM_CHSET_SIZE (CURLFNM_CHARSET_LEN + 15) @@ -334,9 +345,9 @@ static int loop(const unsigned char *pattern, const unsigned char *string, s++; break; } + /* Syntax error in set; mismatch! */ + return CURL_FNMATCH_NOMATCH; - /* Syntax error in set: this must be taken as a regular character. */ - /* FALLTHROUGH */ default: if(*p++ != *s++) return CURL_FNMATCH_NOMATCH; @@ -355,5 +366,31 @@ int Curl_fnmatch(void *ptr, const char *pattern, const char *string) if(!pattern || !string) { return CURL_FNMATCH_FAIL; } - return loop((unsigned char *)pattern, (unsigned char *)string, 5); + return loop((unsigned char *)pattern, (unsigned char *)string, 2); +} +#else +#include <fnmatch.h> +/* + * @unittest: 1307 + */ +int Curl_fnmatch(void *ptr, const char *pattern, const char *string) +{ + int rc; + (void)ptr; /* the argument is specified by the curl_fnmatch_callback + prototype, but not used by Curl_fnmatch() */ + if(!pattern || !string) { + return CURL_FNMATCH_FAIL; + } + rc = fnmatch(pattern, string, 0); + switch(rc) { + case 0: + return CURL_FNMATCH_MATCH; + case FNM_NOMATCH: + return CURL_FNMATCH_NOMATCH; + default: + return CURL_FNMATCH_FAIL; + } + /* not reached */ } + +#endif diff --git a/Utilities/cmcurl/lib/curl_ntlm_core.c b/Utilities/cmcurl/lib/curl_ntlm_core.c index e27cab3..922e85a 100644 --- a/Utilities/cmcurl/lib/curl_ntlm_core.c +++ b/Utilities/cmcurl/lib/curl_ntlm_core.c @@ -557,8 +557,11 @@ CURLcode Curl_ntlm_core_mk_nt_hash(struct Curl_easy *data, unsigned char *ntbuffer /* 21 bytes */) { size_t len = strlen(password); - unsigned char *pw = len ? malloc(len * 2) : strdup(""); + unsigned char *pw; CURLcode result; + if(len > SIZE_T_MAX/2) /* avoid integer overflow */ + return CURLE_OUT_OF_MEMORY; + pw = len ? malloc(len * 2) : strdup(""); if(!pw) return CURLE_OUT_OF_MEMORY; diff --git a/Utilities/cmcurl/lib/curl_sasl.c b/Utilities/cmcurl/lib/curl_sasl.c index e54e487..354bc54 100644 --- a/Utilities/cmcurl/lib/curl_sasl.c +++ b/Utilities/cmcurl/lib/curl_sasl.c @@ -146,7 +146,6 @@ CURLcode Curl_sasl_parse_url_auth_option(struct SASL *sasl, const char *value, size_t len) { CURLcode result = CURLE_OK; - unsigned int mechbit; size_t mechlen; if(!len) @@ -160,7 +159,7 @@ CURLcode Curl_sasl_parse_url_auth_option(struct SASL *sasl, if(!strncmp(value, "*", len)) sasl->prefmech = SASL_AUTH_DEFAULT; else { - mechbit = Curl_sasl_decode_mech(value, len, &mechlen); + unsigned int mechbit = Curl_sasl_decode_mech(value, len, &mechlen); if(mechbit && mechlen == len) sasl->prefmech |= mechbit; else diff --git a/Utilities/cmcurl/lib/curl_setup.h b/Utilities/cmcurl/lib/curl_setup.h index 7e6fd5e..b056126 100644 --- a/Utilities/cmcurl/lib/curl_setup.h +++ b/Utilities/cmcurl/lib/curl_setup.h @@ -221,7 +221,7 @@ /* * Use getaddrinfo to resolve the IPv4 address literal. If the current network - * interface doesn’t support IPv4, but supports IPv6, NAT64, and DNS64, + * interface doesn't support IPv4, but supports IPv6, NAT64, and DNS64, * performing this task will result in a synthesized IPv6 address. */ #ifdef __APPLE__ @@ -246,6 +246,7 @@ # if defined(_UNICODE) && !defined(UNICODE) # define UNICODE # endif +# include <winerror.h> # include <windows.h> # ifdef HAVE_WINSOCK2_H # include <winsock2.h> @@ -815,4 +816,16 @@ endings either CRLF or LF so 't' is appropriate. #define CURL_SA_FAMILY_T unsigned short #endif +/* Some convenience macros to get the larger/smaller value out of two given. + We prefix with CURL to prevent name collisions. */ +#define CURLMAX(x,y) ((x)>(y)?(x):(y)) +#define CURLMIN(x,y) ((x)<(y)?(x):(y)) + +/* Some versions of the Android SDK is missing the declaration */ +#if defined(HAVE_GETPWUID_R) && defined(HAVE_DECL_GETPWUID_R_MISSING) +struct passwd; +int getpwuid_r(uid_t uid, struct passwd *pwd, char *buf, + size_t buflen, struct passwd **result); +#endif + #endif /* HEADER_CURL_SETUP_H */ diff --git a/Utilities/cmcurl/lib/curl_threads.c b/Utilities/cmcurl/lib/curl_threads.c index c1624a9..b8f0cd3 100644 --- a/Utilities/cmcurl/lib/curl_threads.c +++ b/Utilities/cmcurl/lib/curl_threads.c @@ -108,7 +108,8 @@ curl_thread_t Curl_thread_create(unsigned int (CURL_STDCALL *func) (void *), #ifdef _WIN32_WCE t = CreateThread(NULL, 0, func, arg, 0, NULL); #else - t = (curl_thread_t)_beginthreadex(NULL, 0, func, arg, 0, NULL); + uintptr_t thread_handle = _beginthreadex(NULL, 0, func, arg, 0, NULL); + t = (curl_thread_t)thread_handle; #endif if((t == 0) || (t == LongToHandle(-1L))) { #ifdef _WIN32_WCE diff --git a/Utilities/cmcurl/lib/dict.c b/Utilities/cmcurl/lib/dict.c index 4fc8552..408d57b 100644 --- a/Utilities/cmcurl/lib/dict.c +++ b/Utilities/cmcurl/lib/dict.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -95,17 +95,17 @@ static char *unescape_word(struct Curl_easy *data, const char *inputbuff) { char *newp = NULL; char *dictp; - char *ptr; size_t len; - char ch; - int olen = 0; CURLcode result = Curl_urldecode(data, inputbuff, 0, &newp, &len, FALSE); if(!newp || result) return NULL; - dictp = malloc(((size_t)len)*2 + 1); /* add one for terminating zero */ + dictp = malloc(len*2 + 1); /* add one for terminating zero */ if(dictp) { + char *ptr; + char ch; + int olen = 0; /* According to RFC2229 section 2.2, these letters need to be escaped with \[letter] */ for(ptr = newp; diff --git a/Utilities/cmcurl/lib/easy.c b/Utilities/cmcurl/lib/easy.c index 6b91435..027d0be 100644 --- a/Utilities/cmcurl/lib/easy.c +++ b/Utilities/cmcurl/lib/easy.c @@ -113,7 +113,7 @@ static CURLcode win32_init(void) res = WSAStartup(wVersionRequested, &wsaData); if(res != 0) - /* Tell the user that we couldn't find a useable */ + /* Tell the user that we couldn't find a usable */ /* winsock.dll. */ return CURLE_FAILED_INIT; @@ -125,7 +125,7 @@ static CURLcode win32_init(void) if(LOBYTE(wsaData.wVersion) != LOBYTE(wVersionRequested) || HIBYTE(wsaData.wVersion) != HIBYTE(wVersionRequested) ) { - /* Tell the user that we couldn't find a useable */ + /* Tell the user that we couldn't find a usable */ /* winsock.dll. */ WSACleanup(); @@ -661,38 +661,27 @@ static CURLcode easy_transfer(struct Curl_multi *multi) bool done = FALSE; CURLMcode mcode = CURLM_OK; CURLcode result = CURLE_OK; - struct curltime before; - int without_fds = 0; /* count number of consecutive returns from - curl_multi_wait() without any filedescriptors */ while(!done && !mcode) { int still_running = 0; int rc; - before = Curl_now(); mcode = curl_multi_wait(multi, NULL, 0, 1000, &rc); if(!mcode) { if(!rc) { - struct curltime after = Curl_now(); + long sleep_ms; /* If it returns without any filedescriptor instantly, we need to avoid busy-looping during periods where it has nothing particular to wait for */ - if(Curl_timediff(after, before) <= 10) { - without_fds++; - if(without_fds > 2) { - int sleep_ms = without_fds < 10 ? (1 << (without_fds - 1)) : 1000; - Curl_wait_ms(sleep_ms); - } + curl_multi_timeout(multi, &sleep_ms); + if(sleep_ms) { + if(sleep_ms > 1000) + sleep_ms = 1000; + Curl_wait_ms((int)sleep_ms); } - else - /* it wasn't "instant", restart counter */ - without_fds = 0; } - else - /* got file descriptor, restart counter */ - without_fds = 0; mcode = curl_multi_perform(multi, &still_running); } @@ -969,6 +958,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) outcurl->change.referer_alloc = TRUE; } + /* Reinitialize an SSL engine for the new handle + * note: the engine name has already been copied by dupset */ + if(outcurl->set.str[STRING_SSL_ENGINE]) { + if(Curl_ssl_set_engine(outcurl, outcurl->set.str[STRING_SSL_ENGINE])) + goto fail; + } + /* Clone the resolver handle, if present, for the new handle */ if(Curl_resolver_duphandle(&outcurl->state.resolver, data->state.resolver)) diff --git a/Utilities/cmcurl/lib/escape.c b/Utilities/cmcurl/lib/escape.c index b7e2d32..10774f0 100644 --- a/Utilities/cmcurl/lib/escape.c +++ b/Utilities/cmcurl/lib/escape.c @@ -82,7 +82,6 @@ char *curl_easy_escape(struct Curl_easy *data, const char *string, size_t alloc; char *ns; char *testing_ptr = NULL; - unsigned char in; /* we need to treat the characters unsigned */ size_t newlen; size_t strindex = 0; size_t length; @@ -100,7 +99,7 @@ char *curl_easy_escape(struct Curl_easy *data, const char *string, length = alloc-1; while(length--) { - in = *string; + unsigned char in = *string; /* we need to treat the characters unsigned */ if(Curl_isunreserved(in)) /* just copy this */ @@ -150,7 +149,6 @@ CURLcode Curl_urldecode(struct Curl_easy *data, { size_t alloc = (length?length:strlen(string)) + 1; char *ns = malloc(alloc); - unsigned char in; size_t strindex = 0; unsigned long hex; CURLcode result; @@ -159,7 +157,7 @@ CURLcode Curl_urldecode(struct Curl_easy *data, return CURLE_OUT_OF_MEMORY; while(--alloc > 0) { - in = *string; + unsigned char in = *string; if(('%' == in) && (alloc > 2) && ISXDIGIT(string[1]) && ISXDIGIT(string[2])) { /* this is two hexadecimal digits following a '%' */ diff --git a/Utilities/cmcurl/lib/file.c b/Utilities/cmcurl/lib/file.c index db04cc2..e50e988 100644 --- a/Utilities/cmcurl/lib/file.c +++ b/Utilities/cmcurl/lib/file.c @@ -256,8 +256,6 @@ static CURLcode file_upload(struct connectdata *conn) CURLcode result = CURLE_OK; struct Curl_easy *data = conn->data; char *buf = data->state.buffer; - size_t nread; - size_t nwrite; curl_off_t bytecount = 0; struct_stat file_stat; const char *buf2; @@ -306,7 +304,9 @@ static CURLcode file_upload(struct connectdata *conn) } while(!result) { - int readcount; + size_t nread; + size_t nwrite; + size_t readcount; result = Curl_fillreadbuffer(conn, (int)data->set.buffer_size, &readcount); if(result) break; @@ -314,7 +314,7 @@ static CURLcode file_upload(struct connectdata *conn) if(readcount <= 0) /* fix questionable compare error. curlvms */ break; - nread = (size_t)readcount; + nread = readcount; /*skip bytes before resume point*/ if(data->state.resume_from) { @@ -378,7 +378,6 @@ static CURLcode file_do(struct connectdata *conn, bool *done) curl_off_t expected_size = 0; bool size_known; bool fstated = FALSE; - ssize_t nread; struct Curl_easy *data = conn->data; char *buf = data->state.buffer; curl_off_t bytecount = 0; @@ -461,7 +460,7 @@ static CURLcode file_do(struct connectdata *conn, bool *done) return result; /* Adjust the start offset in case we want to get the N last bytes - * of the stream iff the filesize could be determined */ + * of the stream if the filesize could be determined */ if(data->state.resume_from < 0) { if(!fstated) { failf(data, "Can't get the size of file."); @@ -502,6 +501,7 @@ static CURLcode file_do(struct connectdata *conn, bool *done) Curl_pgrsTime(data, TIMER_STARTTRANSFER); while(!result) { + ssize_t nread; /* Don't fill a whole buffer if we want less than all data */ size_t bytestoread; diff --git a/Utilities/cmcurl/lib/formdata.c b/Utilities/cmcurl/lib/formdata.c index f912410..202d930 100644 --- a/Utilities/cmcurl/lib/formdata.c +++ b/Utilities/cmcurl/lib/formdata.c @@ -39,16 +39,13 @@ #include "sendf.h" #include "strdup.h" #include "rand.h" +#include "warnless.h" /* The last 3 #include files should be in this order */ #include "curl_printf.h" #include "curl_memory.h" #include "memdebug.h" -/* What kind of Content-Type to use on un-specified files with unrecognized - extensions. */ -#define HTTPPOST_CONTENTTYPE_DEFAULT "application/octet-stream" - #define HTTPPOST_PTRNAME CURL_HTTPPOST_PTRNAME #define HTTPPOST_FILENAME CURL_HTTPPOST_FILENAME #define HTTPPOST_PTRCONTENTS CURL_HTTPPOST_PTRCONTENTS @@ -305,7 +302,8 @@ CURLFORMcode FormAdd(struct curl_httppost **httppost, * Set the contents property. */ case CURLFORM_PTRCONTENTS: - current_form->flags |= HTTPPOST_PTRCONTENTS; /* fall through */ + current_form->flags |= HTTPPOST_PTRCONTENTS; + /* FALLTHROUGH */ case CURLFORM_COPYCONTENTS: if(current_form->value) return_value = CURL_FORMADD_OPTION_TWICE; @@ -725,7 +723,7 @@ int curl_formget(struct curl_httppost *form, void *arg, while(!result) { char buffer[8192]; - size_t nread = Curl_mime_read(buffer, 1, sizeof buffer, &toppart); + size_t nread = Curl_mime_read(buffer, 1, sizeof(buffer), &toppart); if(!nread) break; @@ -812,7 +810,6 @@ CURLcode Curl_getformdata(struct Curl_easy *data, { CURLcode result = CURLE_OK; curl_mime *form = NULL; - curl_mime *multipart; curl_mimepart *part; struct curl_httppost *file; @@ -831,7 +828,7 @@ CURLcode Curl_getformdata(struct Curl_easy *data, /* Process each top part. */ for(; !result && post; post = post->next) { /* If we have more than a file here, create a mime subpart and fill it. */ - multipart = form; + curl_mime *multipart = form; if(post->more) { part = curl_mime_addpart(form); if(!part) @@ -883,7 +880,8 @@ CURLcode Curl_getformdata(struct Curl_easy *data, compatibility: use of "-" pseudo file name should be avoided. */ result = curl_mime_data_cb(part, (curl_off_t) -1, (curl_read_callback) fread, - (curl_seek_callback) fseek, + CURLX_FUNCTION_CAST(curl_seek_callback, + fseek), NULL, (void *) stdin); } else diff --git a/Utilities/cmcurl/lib/ftp.c b/Utilities/cmcurl/lib/ftp.c index a42c0c7..63f32cf 100644 --- a/Utilities/cmcurl/lib/ftp.c +++ b/Utilities/cmcurl/lib/ftp.c @@ -239,8 +239,8 @@ static void close_secondarysocket(struct connectdata *conn) static void freedirs(struct ftp_conn *ftpc) { - int i; if(ftpc->dirs) { + int i; for(i = 0; i < ftpc->dirdepth; i++) { free(ftpc->dirs[i]); ftpc->dirs[i] = NULL; @@ -637,8 +637,6 @@ CURLcode Curl_GetFTPResponse(ssize_t *nreadp, /* return number of bytes read */ * line in a response or continue reading. */ curl_socket_t sockfd = conn->sock[FIRSTSOCKET]; - time_t timeout; /* timeout in milliseconds */ - time_t interval_ms; struct Curl_easy *data = conn->data; CURLcode result = CURLE_OK; struct ftp_conn *ftpc = &conn->proto.ftpc; @@ -657,7 +655,8 @@ CURLcode Curl_GetFTPResponse(ssize_t *nreadp, /* return number of bytes read */ while(!*ftpcode && !result) { /* check and reset timeout value every lap */ - timeout = Curl_pp_state_timeout(pp); + time_t timeout = Curl_pp_state_timeout(pp); /* timeout in milliseconds */ + time_t interval_ms; if(timeout <= 0) { failf(data, "FTP response timeout"); @@ -1589,7 +1588,6 @@ static CURLcode ftp_state_ul_setup(struct connectdata *conn, struct FTP *ftp = conn->data->req.protop; struct Curl_easy *data = conn->data; struct ftp_conn *ftpc = &conn->proto.ftpc; - int seekerr = CURL_SEEKFUNC_OK; if((data->state.resume_from && !sizechecked) || ((data->state.resume_from > 0) && sizechecked)) { @@ -1605,6 +1603,7 @@ static CURLcode ftp_state_ul_setup(struct connectdata *conn, /* 3. pass file-size number of bytes in the source file */ /* 4. lower the infilesize counter */ /* => transfer as usual */ + int seekerr = CURL_SEEKFUNC_OK; if(data->state.resume_from < 0) { /* Got no given size to start from, figure it out */ @@ -2782,7 +2781,6 @@ static CURLcode ftp_statemach_act(struct connectdata *conn) char *ptr = &data->state.buffer[4]; /* start on the first letter */ const size_t buf_size = data->set.buffer_size; char *dir; - char *store; bool entry_extracted = FALSE; dir = malloc(nread + 1); @@ -2805,6 +2803,7 @@ static CURLcode ftp_statemach_act(struct connectdata *conn) if('\"' == *ptr) { /* it started good */ + char *store; ptr++; for(store = dir; *ptr;) { if('\"' == *ptr) { @@ -3993,8 +3992,7 @@ CURLcode Curl_ftpsend(struct connectdata *conn, const char *cmd) break; if(conn->data->set.verbose) - Curl_debug(conn->data, CURLINFO_HEADER_OUT, - sptr, (size_t)bytes_written, conn); + Curl_debug(conn->data, CURLINFO_HEADER_OUT, sptr, (size_t)bytes_written); if(bytes_written != (ssize_t)write_len) { write_len -= bytes_written; @@ -4385,7 +4383,6 @@ static CURLcode ftp_setup_connection(struct connectdata *conn) { struct Curl_easy *data = conn->data; char *type; - char command; struct FTP *ftp; conn->data->req.protop = ftp = malloc(sizeof(struct FTP)); @@ -4403,6 +4400,7 @@ static CURLcode ftp_setup_connection(struct connectdata *conn) type = strstr(conn->host.rawalloc, ";type="); if(type) { + char command; *type = 0; /* it was in the middle of the hostname */ command = Curl_raw_toupper(type[6]); conn->bits.type_set = TRUE; diff --git a/Utilities/cmcurl/lib/getinfo.c b/Utilities/cmcurl/lib/getinfo.c index d280eeb..14b4562 100644 --- a/Utilities/cmcurl/lib/getinfo.c +++ b/Utilities/cmcurl/lib/getinfo.c @@ -281,6 +281,28 @@ static CURLcode getinfo_offt(struct Curl_easy *data, CURLINFO info, *param_offt = (data->progress.flags & PGRS_UL_SIZE_KNOWN)? data->progress.size_ul:-1; break; + case CURLINFO_TOTAL_TIME_T: + *param_offt = data->progress.timespent; + break; + case CURLINFO_NAMELOOKUP_TIME_T: + *param_offt = data->progress.t_nslookup; + break; + case CURLINFO_CONNECT_TIME_T: + *param_offt = data->progress.t_connect; + break; + case CURLINFO_APPCONNECT_TIME_T: + *param_offt = data->progress.t_appconnect; + break; + case CURLINFO_PRETRANSFER_TIME_T: + *param_offt = data->progress.t_pretransfer; + break; + case CURLINFO_STARTTRANSFER_TIME_T: + *param_offt = data->progress.t_starttransfer; + break; + case CURLINFO_REDIRECT_TIME_T: + *param_offt = data->progress.t_redirect; + break; + default: return CURLE_UNKNOWN_OPTION; } diff --git a/Utilities/cmcurl/lib/gopher.c b/Utilities/cmcurl/lib/gopher.c index b7c31b6..3ecee9b 100644 --- a/Utilities/cmcurl/lib/gopher.c +++ b/Utilities/cmcurl/lib/gopher.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -89,22 +89,15 @@ static CURLcode gopher_do(struct connectdata *conn, bool *done) /* Create selector. Degenerate cases: / and /1 => convert to "" */ if(strlen(path) <= 2) { sel = (char *)""; - len = (int)strlen(sel); + len = strlen(sel); } else { char *newp; - size_t j, i; /* Otherwise, drop / and the first character (i.e., item type) ... */ newp = path; newp += 2; - /* ... then turn ? into TAB for search servers, Veronica, etc. ... */ - j = strlen(newp); - for(i = 0; i<j; i++) - if(newp[i] == '?') - newp[i] = '\x09'; - /* ... and finally unescape */ result = Curl_urldecode(data, newp, 0, &sel, &len, FALSE); if(result) diff --git a/Utilities/cmcurl/lib/hash.c b/Utilities/cmcurl/lib/hash.c index 15a128f..421d68f 100644 --- a/Utilities/cmcurl/lib/hash.c +++ b/Utilities/cmcurl/lib/hash.c @@ -60,8 +60,6 @@ Curl_hash_init(struct curl_hash *h, comp_function comparator, curl_hash_dtor dtor) { - int i; - if(!slots || !hfunc || !comparator ||!dtor) { return 1; /* failure */ } @@ -74,6 +72,7 @@ Curl_hash_init(struct curl_hash *h, h->table = malloc(slots * sizeof(struct curl_llist)); if(h->table) { + int i; for(i = 0; i < slots; ++i) Curl_llist_init(&h->table[i], (curl_llist_dtor) hash_element_dtor); return 0; /* fine */ @@ -140,11 +139,10 @@ Curl_hash_add(struct curl_hash *h, void *key, size_t key_len, void *p) int Curl_hash_delete(struct curl_hash *h, void *key, size_t key_len) { struct curl_llist_element *le; - struct curl_hash_element *he; struct curl_llist *l = FETCH_LIST(h, key, key_len); for(le = l->head; le; le = le->next) { - he = le->ptr; + struct curl_hash_element *he = le->ptr; if(h->comp_func(he->key, he->key_len, key, key_len)) { Curl_llist_remove(l, le, (void *) h); --h->size; @@ -162,13 +160,12 @@ void * Curl_hash_pick(struct curl_hash *h, void *key, size_t key_len) { struct curl_llist_element *le; - struct curl_hash_element *he; struct curl_llist *l; if(h) { l = FETCH_LIST(h, key, key_len); for(le = l->head; le; le = le->next) { - he = le->ptr; + struct curl_hash_element *he = le->ptr; if(h->comp_func(he->key, he->key_len, key, key_len)) { return he->ptr; } @@ -291,7 +288,6 @@ void Curl_hash_start_iterate(struct curl_hash *hash, struct curl_hash_element * Curl_hash_next_element(struct curl_hash_iterator *iter) { - int i; struct curl_hash *h = iter->hash; /* Get the next element in the current list, if any */ @@ -300,6 +296,7 @@ Curl_hash_next_element(struct curl_hash_iterator *iter) /* If we have reached the end of the list, find the next one */ if(!iter->current_element) { + int i; for(i = iter->slot_index; i < h->slots; i++) { if(h->table[i].head) { iter->current_element = h->table[i].head; diff --git a/Utilities/cmcurl/lib/hmac.c b/Utilities/cmcurl/lib/hmac.c index dae9505..bf49ebe 100644 --- a/Utilities/cmcurl/lib/hmac.c +++ b/Utilities/cmcurl/lib/hmac.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -58,7 +58,7 @@ Curl_HMAC_init(const HMAC_params * hashparams, unsigned char b; /* Create HMAC context. */ - i = sizeof *ctxt + 2 * hashparams->hmac_ctxtsize + + i = sizeof(*ctxt) + 2 * hashparams->hmac_ctxtsize + hashparams->hmac_resultlen; ctxt = malloc(i); diff --git a/Utilities/cmcurl/lib/hostasyn.c b/Utilities/cmcurl/lib/hostasyn.c index 7b6e856..e7b399e 100644 --- a/Utilities/cmcurl/lib/hostasyn.c +++ b/Utilities/cmcurl/lib/hostasyn.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -131,7 +131,7 @@ CURLcode Curl_async_resolved(struct connectdata *conn, if(result) /* We're not allowed to return failure with memory left allocated in the connectdata struct, free those here */ - Curl_disconnect(conn, FALSE); /* close the connection */ + Curl_disconnect(conn->data, conn, TRUE); /* close the connection */ return result; } diff --git a/Utilities/cmcurl/lib/hostip.c b/Utilities/cmcurl/lib/hostip.c index c2f9def..bc20f71 100644 --- a/Utilities/cmcurl/lib/hostip.c +++ b/Utilities/cmcurl/lib/hostip.c @@ -907,7 +907,9 @@ CURLcode Curl_loadhostpairs(struct Curl_easy *data) char *entry_id; size_t entry_len; char address[64]; +#if !defined(CURL_DISABLE_VERBOSE_STRINGS) char *addresses = NULL; +#endif char *addr_begin; char *addr_end; char *port_ptr; @@ -930,7 +932,9 @@ CURLcode Curl_loadhostpairs(struct Curl_easy *data) goto err; port = (int)tmp_port; +#if !defined(CURL_DISABLE_VERBOSE_STRINGS) addresses = end_ptr + 1; +#endif while(*end_ptr) { size_t alen; @@ -1010,24 +1014,28 @@ CURLcode Curl_loadhostpairs(struct Curl_easy *data) /* See if its already in our dns cache */ dns = Curl_hash_pick(data->dns.hostcache, entry_id, entry_len + 1); + if(dns) { + infof(data, "RESOLVE %s:%d is - old addresses discarded!\n", + hostname, port); + /* delete old entry entry, there are two reasons for this + 1. old entry may have different addresses. + 2. even if entry with correct addresses is already in the cache, + but if it is close to expire, then by the time next http + request is made, it can get expired and pruned because old + entry is not necessarily marked as added by CURLOPT_RESOLVE. */ + + Curl_hash_delete(data->dns.hostcache, entry_id, entry_len + 1); + } /* free the allocated entry_id again */ free(entry_id); - if(!dns) { - /* if not in the cache already, put this host in the cache */ - dns = Curl_cache_addr(data, head, hostname, port); - if(dns) { - dns->timestamp = 0; /* mark as added by CURLOPT_RESOLVE */ - /* release the returned reference; the cache itself will keep the - * entry alive: */ - dns->inuse--; - } - } - else { - /* this is a duplicate, free it again */ - infof(data, "RESOLVE %s:%d is already cached, %s not stored!\n", - hostname, port, addresses); - Curl_freeaddrinfo(head); + /* put this new host in the cache */ + dns = Curl_cache_addr(data, head, hostname, port); + if(dns) { + dns->timestamp = 0; /* mark as added by CURLOPT_RESOLVE */ + /* release the returned reference; the cache itself will keep the + * entry alive: */ + dns->inuse--; } if(data->share) diff --git a/Utilities/cmcurl/lib/hostip6.c b/Utilities/cmcurl/lib/hostip6.c index 7c9988f..3bf47b4 100644 --- a/Utilities/cmcurl/lib/hostip6.c +++ b/Utilities/cmcurl/lib/hostip6.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -59,39 +59,6 @@ #include "curl_memory.h" #include "memdebug.h" -#if defined(CURLDEBUG) && defined(HAVE_GETNAMEINFO) -/* These are strictly for memory tracing and are using the same style as the - * family otherwise present in memdebug.c. I put these ones here since they - * require a bunch of structs I didn't want to include in memdebug.c - */ - -/* - * For CURLRES_ARS, this should be written using ares_gethostbyaddr() - * (ignoring the fact c-ares doesn't return 'serv'). - */ - -int curl_dogetnameinfo(GETNAMEINFO_QUAL_ARG1 GETNAMEINFO_TYPE_ARG1 sa, - GETNAMEINFO_TYPE_ARG2 salen, - char *host, GETNAMEINFO_TYPE_ARG46 hostlen, - char *serv, GETNAMEINFO_TYPE_ARG46 servlen, - GETNAMEINFO_TYPE_ARG7 flags, - int line, const char *source) -{ - int res = (getnameinfo)(sa, salen, - host, hostlen, - serv, servlen, - flags); - if(0 == res) - /* success */ - curl_memlog("GETNAME %s:%d getnameinfo()\n", - source, line); - else - curl_memlog("GETNAME %s:%d getnameinfo() failed = %d\n", - source, line, res); - return res; -} -#endif /* defined(CURLDEBUG) && defined(HAVE_GETNAMEINFO) */ - /* * Curl_ipv6works() returns TRUE if IPv6 seems to work. */ diff --git a/Utilities/cmcurl/lib/http.c b/Utilities/cmcurl/lib/http.c index ff1d681..e727ed8 100644 --- a/Utilities/cmcurl/lib/http.c +++ b/Utilities/cmcurl/lib/http.c @@ -158,18 +158,20 @@ CURLcode Curl_http_setup_conn(struct connectdata *conn) /* allocate the HTTP-specific struct for the Curl_easy, only to survive during this request */ struct HTTP *http; - DEBUGASSERT(conn->data->req.protop == NULL); + struct Curl_easy *data = conn->data; + DEBUGASSERT(data->req.protop == NULL); http = calloc(1, sizeof(struct HTTP)); if(!http) return CURLE_OUT_OF_MEMORY; Curl_mime_initpart(&http->form, conn->data); - conn->data->req.protop = http; - - Curl_http2_setup_conn(conn); - Curl_http2_setup_req(conn->data); + data->req.protop = http; + if(!CONN_INUSE(conn)) + /* if not alredy multi-using, setup connection details */ + Curl_http2_setup_conn(conn); + Curl_http2_setup_req(data); return CURLE_OK; } @@ -310,22 +312,49 @@ static CURLcode http_output_basic(struct connectdata *conn, bool proxy) return result; } +/* + * http_output_bearer() sets up an Authorization: header + * for HTTP Bearer authentication. + * + * Returns CURLcode. + */ +static CURLcode http_output_bearer(struct connectdata *conn) +{ + char **userp; + CURLcode result = CURLE_OK; + + userp = &conn->allocptr.userpwd; + free(*userp); + *userp = aprintf("Authorization: Bearer %s\r\n", + conn->oauth_bearer); + + if(!*userp) { + result = CURLE_OUT_OF_MEMORY; + goto fail; + } + + fail: + return result; +} + /* pickoneauth() selects the most favourable authentication method from the * ones available and the ones we want. * * return TRUE if one was picked */ -static bool pickoneauth(struct auth *pick) +static bool pickoneauth(struct auth *pick, unsigned long mask) { bool picked; /* only deal with authentication we want */ - unsigned long avail = pick->avail & pick->want; + unsigned long avail = pick->avail & pick->want & mask; picked = TRUE; /* The order of these checks is highly relevant, as this will be the order of preference in case of the existence of multiple accepted types. */ if(avail & CURLAUTH_NEGOTIATE) pick->picked = CURLAUTH_NEGOTIATE; + else if(avail & CURLAUTH_BEARER) + pick->picked = CURLAUTH_BEARER; else if(avail & CURLAUTH_DIGEST) pick->picked = CURLAUTH_DIGEST; else if(avail & CURLAUTH_NTLM) @@ -479,6 +508,10 @@ CURLcode Curl_http_auth_act(struct connectdata *conn) bool pickhost = FALSE; bool pickproxy = FALSE; CURLcode result = CURLE_OK; + unsigned long authmask = ~0ul; + + if(!conn->oauth_bearer) + authmask &= (unsigned long)~CURLAUTH_BEARER; if(100 <= data->req.httpcode && 199 >= data->req.httpcode) /* this is a transient response code, ignore */ @@ -487,17 +520,18 @@ CURLcode Curl_http_auth_act(struct connectdata *conn) if(data->state.authproblem) return data->set.http_fail_on_error?CURLE_HTTP_RETURNED_ERROR:CURLE_OK; - if(conn->bits.user_passwd && + if((conn->bits.user_passwd || conn->oauth_bearer) && ((data->req.httpcode == 401) || (conn->bits.authneg && data->req.httpcode < 300))) { - pickhost = pickoneauth(&data->state.authhost); + pickhost = pickoneauth(&data->state.authhost, authmask); if(!pickhost) data->state.authproblem = TRUE; } if(conn->bits.proxy_user_passwd && ((data->req.httpcode == 407) || (conn->bits.authneg && data->req.httpcode < 300))) { - pickproxy = pickoneauth(&data->state.authproxy); + pickproxy = pickoneauth(&data->state.authproxy, + authmask & ~CURLAUTH_BEARER); if(!pickproxy) data->state.authproblem = TRUE; } @@ -628,6 +662,20 @@ output_auth_headers(struct connectdata *conn, functions work that way */ authstatus->done = TRUE; } + if(authstatus->picked == CURLAUTH_BEARER) { + /* Bearer */ + if((!proxy && conn->oauth_bearer && + !Curl_checkheaders(conn, "Authorization:"))) { + auth = "Bearer"; + result = http_output_bearer(conn); + if(result) + return result; + } + + /* NOTE: this function should set 'done' TRUE, as the other auth + functions work that way */ + authstatus->done = TRUE; + } if(auth) { infof(data, "%s auth using %s with user '%s'\n", @@ -674,7 +722,7 @@ Curl_http_output_auth(struct connectdata *conn, authproxy = &data->state.authproxy; if((conn->bits.httpproxy && conn->bits.proxy_user_passwd) || - conn->bits.user_passwd) + conn->bits.user_passwd || conn->oauth_bearer) /* continue please */; else { authhost->done = TRUE; @@ -883,6 +931,18 @@ CURLcode Curl_http_input_auth(struct connectdata *conn, bool proxy, data->state.authproblem = TRUE; } } + else + if(checkprefix("Bearer", auth)) { + *availp |= CURLAUTH_BEARER; + authp->avail |= CURLAUTH_BEARER; + if(authp->picked == CURLAUTH_BEARER) { + /* We asked for Bearer authentication but got a 40X back + anyway, which basically means our token isn't valid. */ + authp->avail = CURLAUTH_NONE; + infof(data, "Authentication problem. Ignoring this.\n"); + data->state.authproblem = TRUE; + } + } /* there may be multiple methods on one line, so keep reading */ while(*auth && *auth != ',') /* read up to the next comma */ @@ -1063,7 +1123,8 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, CURLcode result; char *ptr; size_t size; - struct HTTP *http = conn->data->req.protop; + struct Curl_easy *data = conn->data; + struct HTTP *http = data->req.protop; size_t sendsize; curl_socket_t sockfd; size_t headersize; @@ -1083,7 +1144,7 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, DEBUGASSERT(size > included_body_bytes); - result = Curl_convert_to_network(conn->data, ptr, headersize); + result = Curl_convert_to_network(data, ptr, headersize); /* Curl_convert_to_network calls failf if unsuccessful */ if(result) { /* conversion failed, free memory and return to the caller */ @@ -1108,8 +1169,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, must copy the data to the uploadbuffer first, since that is the buffer we will be using if this send is retried later. */ - memcpy(conn->data->state.uploadbuffer, ptr, sendsize); - ptr = conn->data->state.uploadbuffer; + result = Curl_get_upload_buffer(data); + if(result) { + /* malloc failed, free memory and return to the caller */ + Curl_add_buffer_free(in); + return result; + } + memcpy(data->state.ulbuf, ptr, sendsize); + ptr = data->state.ulbuf; } else sendsize = size; @@ -1126,14 +1193,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, size_t headlen = (size_t)amount>headersize ? headersize : (size_t)amount; size_t bodylen = amount - headlen; - if(conn->data->set.verbose) { + if(data->set.verbose) { /* this data _may_ contain binary stuff */ - Curl_debug(conn->data, CURLINFO_HEADER_OUT, ptr, headlen, conn); + Curl_debug(data, CURLINFO_HEADER_OUT, ptr, headlen); if(bodylen) { /* there was body data sent beyond the initial header part, pass that on to the debug callback too */ - Curl_debug(conn->data, CURLINFO_DATA_OUT, - ptr + headlen, bodylen, conn); + Curl_debug(data, CURLINFO_DATA_OUT, + ptr + headlen, bodylen); } } @@ -1157,14 +1224,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, ptr = in->buffer + amount; /* backup the currently set pointers */ - http->backup.fread_func = conn->data->state.fread_func; - http->backup.fread_in = conn->data->state.in; + http->backup.fread_func = data->state.fread_func; + http->backup.fread_in = data->state.in; http->backup.postdata = http->postdata; http->backup.postsize = http->postsize; /* set the new pointers for the request-sending */ - conn->data->state.fread_func = (curl_read_callback)readmoredata; - conn->data->state.in = (void *)conn; + data->state.fread_func = (curl_read_callback)readmoredata; + data->state.in = (void *)conn; http->postdata = ptr; http->postsize = (curl_off_t)size; @@ -1223,7 +1290,6 @@ CURLcode Curl_add_bufferf(Curl_send_buffer *in, const char *fmt, ...) CURLcode Curl_add_buffer(Curl_send_buffer *in, const void *inptr, size_t size) { char *new_rb; - size_t new_size; if(~size < in->size_used) { /* If resulting used size of send buffer would wrap size_t, cleanup @@ -1236,10 +1302,10 @@ CURLcode Curl_add_buffer(Curl_send_buffer *in, const void *inptr, size_t size) if(!in->buffer || ((in->size_used + size) > (in->size_max - 1))) { - /* If current buffer size isn't enough to hold the result, use a buffer size that doubles the required size. If this new size would wrap size_t, then just use the largest possible one */ + size_t new_size; if((size > (size_t)-1 / 2) || (in->size_used > (size_t)-1 / 2) || (~(size * 2) < (in->size_used * 2))) @@ -1406,7 +1472,7 @@ static CURLcode add_haproxy_protocol_header(struct connectdata *conn) } snprintf(proxy_header, - sizeof proxy_header, + sizeof(proxy_header), "PROXY %s %s %s %li %li\r\n", tcp_version, conn->data->info.conn_local_ip, @@ -1574,7 +1640,6 @@ static CURLcode expect100(struct Curl_easy *data, Curl_send_buffer *req_buffer) { CURLcode result = CURLE_OK; - const char *ptr; data->state.expect100header = FALSE; /* default to false unless it is set to TRUE below */ if(use_http_1_1plus(data, conn) && @@ -1582,7 +1647,7 @@ static CURLcode expect100(struct Curl_easy *data, /* if not doing HTTP 1.0 or version 2, or disabled explicitly, we add an Expect: 100-continue to the headers which actually speeds up post operations (as there is one packet coming back from the web server) */ - ptr = Curl_checkheaders(conn, "Expect"); + const char *ptr = Curl_checkheaders(conn, "Expect"); if(ptr) { data->state.expect100header = Curl_compareheader(ptr, "Expect:", "100-continue"); @@ -1819,7 +1884,6 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) const char *httpstring; Curl_send_buffer *req_buffer; curl_off_t postsize = 0; /* curl_off_t to handle large file sizes */ - int seekerr = CURL_SEEKFUNC_CANTSEEK; /* Always consider the DO phase done after this function call, even if there may be parts of the request that is not yet sent, since we can deal with @@ -1863,6 +1927,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) } http = data->req.protop; + DEBUGASSERT(http); if(!data->state.this_is_a_follow) { /* Free to avoid leaking memory on multiple requests*/ @@ -2088,7 +2153,6 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) else { /* If the host begins with '[', we start searching for the port after the bracket has been closed */ - int startsearch = 0; if(*cookiehost == '[') { char *closingbracket; /* since the 'cookiehost' is an allocated memory area that will be @@ -2099,6 +2163,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) *closingbracket = 0; } else { + int startsearch = 0; char *colon = strchr(cookiehost + startsearch, ':'); if(colon) *colon = 0; /* The host must not include an embedded port number */ @@ -2244,6 +2309,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) /* Now, let's read off the proper amount of bytes from the input. */ + int seekerr = CURL_SEEKFUNC_CANTSEEK; if(conn->seek_func) { Curl_set_in_callback(data, true); seekerr = conn->seek_func(conn->seek_client, data->state.resume_from, @@ -2828,6 +2894,8 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) data->req.exp100 = EXP100_SEND_DATA; /* already sent */ Curl_expire_done(data, EXPIRE_100_TIMEOUT); } + else + data->req.writebytecount = http->writebytecount; } if((conn->httpversion == 20) && data->req.upload_chunky) @@ -2838,17 +2906,32 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) return result; } +typedef enum { + STATUS_UNKNOWN, /* not enough data to tell yet */ + STATUS_DONE, /* a status line was read */ + STATUS_BAD /* not a status line */ +} statusline; + + +/* Check a string for a prefix. Check no more than 'len' bytes */ +static bool checkprefixmax(const char *prefix, const char *buffer, size_t len) +{ + size_t ch = CURLMIN(strlen(prefix), len); + return curl_strnequal(prefix, buffer, ch); +} + /* * checkhttpprefix() * * Returns TRUE if member of the list matches prefix of string */ -static bool +static statusline checkhttpprefix(struct Curl_easy *data, - const char *s) + const char *s, size_t len) { struct curl_slist *head = data->set.http200aliases; - bool rc = FALSE; + statusline rc = STATUS_BAD; + statusline onmatch = len >= 5? STATUS_DONE : STATUS_UNKNOWN; #ifdef CURL_DOES_CONVERSIONS /* convert from the network encoding using a scratch area */ char *scratch = strdup(s); @@ -2865,15 +2948,15 @@ checkhttpprefix(struct Curl_easy *data, #endif /* CURL_DOES_CONVERSIONS */ while(head) { - if(checkprefix(head->data, s)) { - rc = TRUE; + if(checkprefixmax(head->data, s, len)) { + rc = onmatch; break; } head = head->next; } - if(!rc && (checkprefix("HTTP/", s))) - rc = TRUE; + if((rc != STATUS_DONE) && (checkprefixmax("HTTP/", s, len))) + rc = onmatch; #ifdef CURL_DOES_CONVERSIONS free(scratch); @@ -2882,11 +2965,12 @@ checkhttpprefix(struct Curl_easy *data, } #ifndef CURL_DISABLE_RTSP -static bool +static statusline checkrtspprefix(struct Curl_easy *data, - const char *s) + const char *s, size_t len) { - bool result = FALSE; + statusline result = STATUS_BAD; + statusline onmatch = len >= 5? STATUS_DONE : STATUS_UNKNOWN; #ifdef CURL_DOES_CONVERSIONS /* convert from the network encoding using a scratch area */ @@ -2899,30 +2983,31 @@ checkrtspprefix(struct Curl_easy *data, /* Curl_convert_from_network calls failf if unsuccessful */ result = FALSE; /* can't return CURLE_foobar so return FALSE */ } - else - result = checkprefix("RTSP/", scratch)? TRUE: FALSE; + else if(checkprefixmax("RTSP/", scratch, len)) + result = onmatch; free(scratch); #else (void)data; /* unused */ - result = checkprefix("RTSP/", s)? TRUE: FALSE; + if(checkprefixmax("RTSP/", s, len)) + result = onmatch; #endif /* CURL_DOES_CONVERSIONS */ return result; } #endif /* CURL_DISABLE_RTSP */ -static bool +static statusline checkprotoprefix(struct Curl_easy *data, struct connectdata *conn, - const char *s) + const char *s, size_t len) { #ifndef CURL_DISABLE_RTSP if(conn->handler->protocol & CURLPROTO_RTSP) - return checkrtspprefix(data, s); + return checkrtspprefix(data, s, len); #else (void)conn; #endif /* CURL_DISABLE_RTSP */ - return checkhttpprefix(data, s); + return checkhttpprefix(data, s, len); } /* @@ -2939,7 +3024,7 @@ static CURLcode header_append(struct Curl_easy *data, /* The reason to have a max limit for this is to avoid the risk of a bad server feeding libcurl with a never-ending header that will cause reallocs infinitely */ - failf(data, "Rejected %zd bytes header (max is %d)!", newsize, + failf(data, "Rejected %zu bytes header (max is %d)!", newsize, CURL_MAX_HTTP_HEADER); return CURLE_OUT_OF_MEMORY; } @@ -3036,12 +3121,15 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, if(result) return result; - if(!k->headerline && (k->hbuflen>5)) { - /* make a first check that this looks like a protocol header */ - if(!checkprotoprefix(data, conn, data->state.headerbuff)) { + if(!k->headerline) { + /* check if this looks like a protocol header */ + statusline st = checkprotoprefix(data, conn, data->state.headerbuff, + k->hbuflen); + if(st == STATUS_BAD) { /* this is not the beginning of a protocol first header line */ k->header = FALSE; k->badheader = HEADER_ALLBAD; + streamclose(conn, "bad HTTP: No end-of-message indicator"); break; } } @@ -3070,8 +3158,10 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, if(!k->headerline) { /* the first read header */ - if((k->hbuflen>5) && - !checkprotoprefix(data, conn, data->state.headerbuff)) { + statusline st = checkprotoprefix(data, conn, data->state.headerbuff, + k->hbuflen); + if(st == STATUS_BAD) { + streamclose(conn, "bad HTTP: No end-of-message indicator"); /* this is not the beginning of a protocol first header line */ k->header = FALSE; if(*nread) @@ -3354,7 +3444,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, if(data->set.verbose) Curl_debug(data, CURLINFO_HEADER_IN, - k->str_start, headerlen, conn); + k->str_start, headerlen); break; /* exit header line loop */ } @@ -3440,7 +3530,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, compare header line against list of aliases */ if(!nc) { - if(checkhttpprefix(data, k->p)) { + if(checkhttpprefix(data, k->p, k->hbuflen) == STATUS_DONE) { nc = 1; k->httpcode = 200; conn->httpversion = 10; @@ -3487,21 +3577,18 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, * depending on how authentication is working. Other codes * are definitely errors, so give up here. */ - if(data->set.http_fail_on_error && (k->httpcode >= 400) && + if(data->state.resume_from && data->set.httpreq == HTTPREQ_GET && + k->httpcode == 416) { + /* "Requested Range Not Satisfiable", just proceed and + pretend this is no error */ + k->ignorebody = TRUE; /* Avoid appending error msg to good data. */ + } + else if(data->set.http_fail_on_error && (k->httpcode >= 400) && ((k->httpcode != 401) || !conn->bits.user_passwd) && ((k->httpcode != 407) || !conn->bits.proxy_user_passwd) ) { - - if(data->state.resume_from && - (data->set.httpreq == HTTPREQ_GET) && - (k->httpcode == 416)) { - /* "Requested Range Not Satisfiable", just proceed and - pretend this is no error */ - } - else { - /* serious error, go home! */ - print_http_error(data); - return CURLE_HTTP_RETURNED_ERROR; - } + /* serious error, go home! */ + print_http_error(data); + return CURLE_HTTP_RETURNED_ERROR; } if(conn->httpversion == 10) { @@ -3812,8 +3899,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, writetype |= CLIENTWRITE_BODY; if(data->set.verbose) - Curl_debug(data, CURLINFO_HEADER_IN, - k->p, (size_t)k->hbuflen, conn); + Curl_debug(data, CURLINFO_HEADER_IN, k->p, (size_t)k->hbuflen); result = Curl_client_write(conn, writetype, k->p, k->hbuflen); if(result) diff --git a/Utilities/cmcurl/lib/http2.c b/Utilities/cmcurl/lib/http2.c index da001df..d769193 100644 --- a/Utilities/cmcurl/lib/http2.c +++ b/Utilities/cmcurl/lib/http2.c @@ -42,7 +42,6 @@ #include "memdebug.h" #define H2_BUFSIZE 32768 -#define MIN(x,y) ((x)<(y)?(x):(y)) #if (NGHTTP2_VERSION_NUM < 0x010000) #error too old nghttp2 version, upgrade! @@ -154,6 +153,11 @@ static void http2_stream_free(struct HTTP *http) } } +/* + * Disconnects *a* connection used for HTTP/2. It might be an old one from the + * connection cache and not the "main" one. Don't touch the easy handle! + */ + static CURLcode http2_disconnect(struct connectdata *conn, bool dead_connection) { @@ -164,8 +168,6 @@ static CURLcode http2_disconnect(struct connectdata *conn, nghttp2_session_del(c->h2); Curl_safefree(c->inbuf); - http2_stream_free(conn->data->req.protop); - conn->data->state.drain = 0; H2BUGF(infof(conn->data, "HTTP/2 DISCONNECT done\n")); @@ -345,7 +347,7 @@ const char *Curl_http2_strerror(uint32_t err) "INADEQUATE_SECURITY", /* 0xC */ "HTTP_1_1_REQUIRED" /* 0xD */ }; - return (err < sizeof str / sizeof str[0]) ? str[err] : "unknown"; + return (err < sizeof(str) / sizeof(str[0])) ? str[err] : "unknown"; #else return nghttp2_http2_strerror(err); #endif @@ -368,6 +370,10 @@ static ssize_t send_callback(nghttp2_session *h2, (void)h2; (void)flags; + if(!c->send_underlying) + /* called before setup properly! */ + return NGHTTP2_ERR_CALLBACK_FAILURE; + written = ((Curl_send*)c->send_underlying)(conn, FIRSTSOCKET, data, length, &result); @@ -441,6 +447,28 @@ char *curl_pushheader_byname(struct curl_pushheaders *h, const char *header) return NULL; } +/* + * This specific transfer on this connection has been "drained". + */ +static void drained_transfer(struct Curl_easy *data, + struct http_conn *httpc) +{ + DEBUGASSERT(httpc->drain_total >= data->state.drain); + httpc->drain_total -= data->state.drain; + data->state.drain = 0; +} + +/* + * Mark this transfer to get "drained". + */ +static void drain_this(struct Curl_easy *data, + struct http_conn *httpc) +{ + data->state.drain++; + httpc->drain_total++; + DEBUGASSERT(httpc->drain_total >= data->state.drain); +} + static struct Curl_easy *duphandle(struct Curl_easy *data) { struct Curl_easy *second = curl_easy_duphandle(data); @@ -520,6 +548,7 @@ static int push_promise(struct Curl_easy *data, if(rv) { /* denied, kill off the new handle again */ http2_stream_free(newhandle->req.protop); + newhandle->req.protop = NULL; (void)Curl_close(newhandle); goto fail; } @@ -535,14 +564,22 @@ static int push_promise(struct Curl_easy *data, if(rc) { infof(data, "failed to add handle to multi\n"); http2_stream_free(newhandle->req.protop); + newhandle->req.protop = NULL; Curl_close(newhandle); rv = 1; goto fail; } httpc = &conn->proto.httpc; - nghttp2_session_set_stream_user_data(httpc->h2, - frame->promised_stream_id, newhandle); + rv = nghttp2_session_set_stream_user_data(httpc->h2, + frame->promised_stream_id, + newhandle); + if(rv) { + infof(data, "failed to set user_data for stream %d\n", + frame->promised_stream_id); + DEBUGASSERT(0); + goto fail; + } } else { H2BUGF(infof(data, "Got PUSH_PROMISE, ignore it!\n")); @@ -581,7 +618,7 @@ static int on_frame_recv(nghttp2_session *session, const nghttp2_frame *frame, if(max_conn != httpc->settings.max_concurrent_streams) { /* only signal change if the value actually changed */ infof(conn->data, - "Connection state changed (MAX_CONCURRENT_STREAMS == %d)!\n", + "Connection state changed (MAX_CONCURRENT_STREAMS == %u)!\n", httpc->settings.max_concurrent_streams); Curl_multi_connchanged(conn->data->multi); } @@ -640,7 +677,7 @@ static int on_frame_recv(nghttp2_session *session, const nghttp2_frame *frame, Curl_add_buffer(stream->header_recvbuf, "\r\n", 2); left = stream->header_recvbuf->size_used - stream->nread_header_recvbuf; - ncopy = MIN(stream->len, left); + ncopy = CURLMIN(stream->len, left); memcpy(&stream->mem[stream->memlen], stream->header_recvbuf->buffer + stream->nread_header_recvbuf, @@ -653,8 +690,7 @@ static int on_frame_recv(nghttp2_session *session, const nghttp2_frame *frame, stream->len -= ncopy; stream->memlen += ncopy; - data_s->state.drain++; - httpc->drain_total++; + drain_this(data_s, httpc); { /* get the pointer from userp again since it was re-assigned above */ struct connectdata *conn_s = (struct connectdata *)userp; @@ -683,25 +719,6 @@ static int on_frame_recv(nghttp2_session *session, const nghttp2_frame *frame, return 0; } -static int on_invalid_frame_recv(nghttp2_session *session, - const nghttp2_frame *frame, - int lib_error_code, void *userp) -{ - struct Curl_easy *data_s = NULL; - (void)userp; -#if !defined(DEBUG_HTTP2) || defined(CURL_DISABLE_VERBOSE_STRINGS) - (void)lib_error_code; -#endif - - data_s = nghttp2_session_get_stream_user_data(session, frame->hd.stream_id); - if(data_s) { - H2BUGF(infof(data_s, - "on_invalid_frame_recv() was called, error=%d:%s\n", - lib_error_code, nghttp2_strerror(lib_error_code))); - } - return 0; -} - static int on_data_chunk_recv(nghttp2_session *session, uint8_t flags, int32_t stream_id, const uint8_t *data, size_t len, void *userp) @@ -727,14 +744,13 @@ static int on_data_chunk_recv(nghttp2_session *session, uint8_t flags, if(!stream) return NGHTTP2_ERR_CALLBACK_FAILURE; - nread = MIN(stream->len, len); + nread = CURLMIN(stream->len, len); memcpy(&stream->mem[stream->memlen], data, nread); stream->len -= nread; stream->memlen += nread; - data_s->state.drain++; - conn->proto.httpc.drain_total++; + drain_this(data_s, &conn->proto.httpc); /* if we receive data for another handle, wake that up */ if(conn->data != data_s) @@ -768,58 +784,13 @@ static int on_data_chunk_recv(nghttp2_session *session, uint8_t flags, return 0; } -static int before_frame_send(nghttp2_session *session, - const nghttp2_frame *frame, - void *userp) -{ - struct Curl_easy *data_s; - (void)userp; - - data_s = nghttp2_session_get_stream_user_data(session, frame->hd.stream_id); - if(data_s) { - H2BUGF(infof(data_s, "before_frame_send() was called\n")); - } - - return 0; -} -static int on_frame_send(nghttp2_session *session, - const nghttp2_frame *frame, - void *userp) -{ - struct Curl_easy *data_s; - (void)userp; - - data_s = nghttp2_session_get_stream_user_data(session, frame->hd.stream_id); - if(data_s) { - H2BUGF(infof(data_s, "on_frame_send() was called, length = %zd\n", - frame->hd.length)); - } - return 0; -} -static int on_frame_not_send(nghttp2_session *session, - const nghttp2_frame *frame, - int lib_error_code, void *userp) -{ - struct Curl_easy *data_s; - (void)userp; -#if !defined(DEBUG_HTTP2) || defined(CURL_DISABLE_VERBOSE_STRINGS) - (void)lib_error_code; -#endif - - data_s = nghttp2_session_get_stream_user_data(session, frame->hd.stream_id); - if(data_s) { - H2BUGF(infof(data_s, - "on_frame_not_send() was called, lib_error_code = %d\n", - lib_error_code)); - } - return 0; -} static int on_stream_close(nghttp2_session *session, int32_t stream_id, uint32_t error_code, void *userp) { struct Curl_easy *data_s; struct HTTP *stream; struct connectdata *conn = (struct connectdata *)userp; + int rv; (void)session; (void)stream_id; @@ -840,14 +811,19 @@ static int on_stream_close(nghttp2_session *session, int32_t stream_id, return NGHTTP2_ERR_CALLBACK_FAILURE; stream->closed = TRUE; - data_s->state.drain++; httpc = &conn->proto.httpc; - httpc->drain_total++; + drain_this(data_s, httpc); httpc->error_code = error_code; /* remove the entry from the hash as the stream is now gone */ - nghttp2_session_set_stream_user_data(session, stream_id, 0); + rv = nghttp2_session_set_stream_user_data(session, stream_id, 0); + if(rv) { + infof(data_s, "http/2: failed to clear user_data for stream %d!\n", + stream_id); + DEBUGASSERT(0); + } H2BUGF(infof(data_s, "Removed stream %u hash!\n", stream_id)); + stream->stream_id = 0; /* cleared */ } return 0; } @@ -1052,7 +1028,7 @@ static ssize_t data_source_read_callback(nghttp2_session *session, else return NGHTTP2_ERR_INVALID_ARGUMENT; - nread = MIN(stream->upload_len, length); + nread = CURLMIN(stream->upload_len, length); if(nread > 0) { memcpy(buf, stream->upload_mem, nread); stream->upload_mem += nread; @@ -1109,6 +1085,12 @@ void Curl_http2_done(struct connectdata *conn, bool premature) struct HTTP *http = data->req.protop; struct http_conn *httpc = &conn->proto.httpc; + if(!httpc->h2) /* not HTTP/2 ? */ + return; + + if(data->state.drain) + drained_transfer(data, httpc); + if(http->header_recvbuf) { Curl_add_buffer_free(http->header_recvbuf); http->header_recvbuf = NULL; /* clear the pointer */ @@ -1126,15 +1108,24 @@ void Curl_http2_done(struct connectdata *conn, bool premature) if(premature) { /* RST_STREAM */ - nghttp2_submit_rst_stream(httpc->h2, NGHTTP2_FLAG_NONE, http->stream_id, - NGHTTP2_STREAM_CLOSED); + if(!nghttp2_submit_rst_stream(httpc->h2, NGHTTP2_FLAG_NONE, + http->stream_id, NGHTTP2_STREAM_CLOSED)) + (void)nghttp2_session_send(httpc->h2); + if(http->stream_id == httpc->pause_stream_id) { infof(data, "stopped the pause stream!\n"); httpc->pause_stream_id = 0; } } - if(http->stream_id) { - nghttp2_session_set_stream_user_data(httpc->h2, http->stream_id, 0); + /* -1 means unassigned and 0 means cleared */ + if(http->stream_id > 0) { + int rv = nghttp2_session_set_stream_user_data(httpc->h2, + http->stream_id, 0); + if(rv) { + infof(data, "http/2: failed to clear user_data for stream %d!\n", + http->stream_id); + DEBUGASSERT(0); + } http->stream_id = 0; } } @@ -1164,21 +1155,9 @@ CURLcode Curl_http2_init(struct connectdata *conn) /* nghttp2_on_frame_recv_callback */ nghttp2_session_callbacks_set_on_frame_recv_callback (callbacks, on_frame_recv); - /* nghttp2_on_invalid_frame_recv_callback */ - nghttp2_session_callbacks_set_on_invalid_frame_recv_callback - (callbacks, on_invalid_frame_recv); /* nghttp2_on_data_chunk_recv_callback */ nghttp2_session_callbacks_set_on_data_chunk_recv_callback (callbacks, on_data_chunk_recv); - /* nghttp2_before_frame_send_callback */ - nghttp2_session_callbacks_set_before_frame_send_callback - (callbacks, before_frame_send); - /* nghttp2_on_frame_send_callback */ - nghttp2_session_callbacks_set_on_frame_send_callback - (callbacks, on_frame_send); - /* nghttp2_on_frame_not_send_callback */ - nghttp2_session_callbacks_set_on_frame_not_send_callback - (callbacks, on_frame_not_send); /* nghttp2_on_stream_close_callback */ nghttp2_session_callbacks_set_on_stream_close_callback (callbacks, on_stream_close); @@ -1280,7 +1259,7 @@ static int h2_process_pending_input(struct connectdata *conn, if(rv < 0) { failf(data, "h2_process_pending_input: nghttp2_session_mem_recv() returned " - "%d:%s\n", rv, nghttp2_strerror((int)rv)); + "%zd:%s\n", rv, nghttp2_strerror((int)rv)); *err = CURLE_RECV_ERROR; return -1; } @@ -1352,7 +1331,6 @@ CURLcode Curl_http2_done_sending(struct connectdata *conn) return result; } - static ssize_t http2_handle_stream_close(struct connectdata *conn, struct Curl_easy *data, struct HTTP *stream, CURLcode *err) @@ -1365,9 +1343,7 @@ static ssize_t http2_handle_stream_close(struct connectdata *conn, httpc->pause_stream_id = 0; } - DEBUGASSERT(httpc->drain_total >= data->state.drain); - httpc->drain_total -= data->state.drain; - data->state.drain = 0; + drained_transfer(data, httpc); if(httpc->pause_stream_id == 0) { if(h2_process_pending_input(conn, httpc, err) != 0) { @@ -1388,7 +1364,7 @@ static ssize_t http2_handle_stream_close(struct connectdata *conn, return -1; } else if(httpc->error_code != NGHTTP2_NO_ERROR) { - failf(data, "HTTP/2 stream %u was not closed cleanly: %s (err %d)", + failf(data, "HTTP/2 stream %d was not closed cleanly: %s (err %u)", stream->stream_id, Curl_http2_strerror(httpc->error_code), httpc->error_code); *err = CURLE_HTTP2_STREAM; @@ -1396,7 +1372,7 @@ static ssize_t http2_handle_stream_close(struct connectdata *conn, } if(!stream->bodystarted) { - failf(data, "HTTP/2 stream %u was closed cleanly, but before getting " + failf(data, "HTTP/2 stream %d was closed cleanly, but before getting " " all response header fields, treated as error", stream->stream_id); *err = CURLE_HTTP2_STREAM; @@ -1510,7 +1486,7 @@ static ssize_t http2_recv(struct connectdata *conn, int sockindex, /* If there is body data pending for this stream to return, do that */ size_t left = stream->header_recvbuf->size_used - stream->nread_header_recvbuf; - size_t ncopy = MIN(len, left); + size_t ncopy = CURLMIN(len, left); memcpy(mem, stream->header_recvbuf->buffer + stream->nread_header_recvbuf, ncopy); stream->nread_header_recvbuf += ncopy; @@ -1546,13 +1522,13 @@ static ssize_t http2_recv(struct connectdata *conn, int sockindex, } else if(stream->pausedata) { DEBUGASSERT(httpc->pause_stream_id == stream->stream_id); - nread = MIN(len, stream->pauselen); + nread = CURLMIN(len, stream->pauselen); memcpy(mem, stream->pausedata, nread); stream->pausedata += nread; stream->pauselen -= nread; - infof(data, "%zu data bytes written\n", nread); + infof(data, "%zd data bytes written\n", nread); if(stream->pauselen == 0) { H2BUGF(infof(data, "Unpaused by stream %u\n", stream->stream_id)); DEBUGASSERT(httpc->pause_stream_id == stream->stream_id); @@ -1635,7 +1611,7 @@ static ssize_t http2_recv(struct connectdata *conn, int sockindex, rv = nghttp2_session_mem_recv(httpc->h2, (const uint8_t *)inbuf, nread); if(nghttp2_is_fatal((int)rv)) { - failf(data, "nghttp2_session_mem_recv() returned %d:%s\n", + failf(data, "nghttp2_session_mem_recv() returned %zd:%s\n", rv, nghttp2_strerror((int)rv)); *err = CURLE_RECV_ERROR; return -1; @@ -1678,9 +1654,7 @@ static ssize_t http2_recv(struct connectdata *conn, int sockindex, stream->stream_id)); } else if(!stream->closed) { - DEBUGASSERT(httpc->drain_total >= data->state.drain); - httpc->drain_total -= data->state.drain; - data->state.drain = 0; /* this stream is hereby drained */ + drained_transfer(data, httpc); } return retlen; @@ -2040,7 +2014,7 @@ static ssize_t http2_send(struct connectdata *conn, int sockindex, } infof(conn->data, "Using Stream ID: %x (easy handle %p)\n", - stream_id, conn->data); + stream_id, (void *)conn->data); stream->stream_id = stream_id; /* this does not call h2_session_send() since there can not have been any @@ -2154,9 +2128,14 @@ CURLcode Curl_http2_switched(struct connectdata *conn, return CURLE_HTTP2; } - nghttp2_session_set_stream_user_data(httpc->h2, - stream->stream_id, - conn->data); + rv = nghttp2_session_set_stream_user_data(httpc->h2, + stream->stream_id, + data); + if(rv) { + infof(data, "http/2: failed to set user_data for stream %d!\n", + stream->stream_id); + DEBUGASSERT(0); + } } else { populate_settings(conn, httpc); diff --git a/Utilities/cmcurl/lib/http2.h b/Utilities/cmcurl/lib/http2.h index f597c80..21cd9b8 100644 --- a/Utilities/cmcurl/lib/http2.h +++ b/Utilities/cmcurl/lib/http2.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -65,7 +65,7 @@ void Curl_http2_cleanup_dependencies(struct Curl_easy *data); #define Curl_http2_request_upgrade(x,y) CURLE_UNSUPPORTED_PROTOCOL #define Curl_http2_setup(x) CURLE_UNSUPPORTED_PROTOCOL #define Curl_http2_switched(x,y,z) CURLE_UNSUPPORTED_PROTOCOL -#define Curl_http2_setup_conn(x) +#define Curl_http2_setup_conn(x) Curl_nop_stmt #define Curl_http2_setup_req(x) #define Curl_http2_init_state(x) #define Curl_http2_init_userset(x) diff --git a/Utilities/cmcurl/lib/http_ntlm.c b/Utilities/cmcurl/lib/http_ntlm.c index fd5540b..a9b33f9 100644 --- a/Utilities/cmcurl/lib/http_ntlm.c +++ b/Utilities/cmcurl/lib/http_ntlm.c @@ -228,7 +228,7 @@ CURLcode Curl_output_ntlm(struct connectdata *conn, bool proxy) /* connection is already authenticated, * don't send a header in future requests */ ntlm->state = NTLMSTATE_LAST; - /* fall-through */ + /* FALLTHROUGH */ case NTLMSTATE_LAST: Curl_safefree(*allocuserpwd); authp->done = TRUE; diff --git a/Utilities/cmcurl/lib/http_proxy.c b/Utilities/cmcurl/lib/http_proxy.c index e10a488..c8c445b 100644 --- a/Utilities/cmcurl/lib/http_proxy.c +++ b/Utilities/cmcurl/lib/http_proxy.c @@ -188,12 +188,10 @@ static CURLcode CONNECT(struct connectdata *conn, struct SingleRequest *k = &data->req; CURLcode result; curl_socket_t tunnelsocket = conn->sock[sockindex]; - timediff_t check; struct http_connect_state *s = conn->connect_state; #define SELECT_OK 0 #define SELECT_ERROR 1 -#define SELECT_TIMEOUT 2 if(Curl_connect_complete(conn)) return CURLE_OK; /* CONNECT is already completed */ @@ -201,12 +199,13 @@ static CURLcode CONNECT(struct connectdata *conn, conn->bits.proxy_connect_closed = FALSE; do { + timediff_t check; if(TUNNEL_INIT == s->tunnel_state) { /* BEGIN CONNECT PHASE */ char *host_port; Curl_send_buffer *req_buffer; - infof(data, "Establish HTTP proxy tunnel to %s:%hu\n", + infof(data, "Establish HTTP proxy tunnel to %s:%d\n", hostname, remote_port); /* This only happens if we've looped here due to authentication @@ -419,7 +418,7 @@ static CURLcode CONNECT(struct connectdata *conn, /* output debug if that is requested */ if(data->set.verbose) Curl_debug(data, CURLINFO_HEADER_IN, - s->line_start, (size_t)s->perline, conn); + s->line_start, (size_t)s->perline); if(!data->set.suppress_connect_headers) { /* send the header to the callback */ diff --git a/Utilities/cmcurl/lib/imap.c b/Utilities/cmcurl/lib/imap.c index cf278a2..942fe7d 100644 --- a/Utilities/cmcurl/lib/imap.c +++ b/Utilities/cmcurl/lib/imap.c @@ -609,7 +609,6 @@ static CURLcode imap_perform_list(struct connectdata *conn) CURLcode result = CURLE_OK; struct Curl_easy *data = conn->data; struct IMAP *imap = data->req.protop; - char *mailbox; if(imap->custom) /* Send the custom request */ @@ -617,7 +616,8 @@ static CURLcode imap_perform_list(struct connectdata *conn) imap->custom_params ? imap->custom_params : ""); else { /* Make sure the mailbox is in the correct atom format if necessary */ - mailbox = imap->mailbox ? imap_atom(imap->mailbox, true) : strdup(""); + char *mailbox = imap->mailbox ? imap_atom(imap->mailbox, true) + : strdup(""); if(!mailbox) return CURLE_OUT_OF_MEMORY; @@ -854,7 +854,6 @@ static CURLcode imap_state_capability_resp(struct connectdata *conn, struct Curl_easy *data = conn->data; struct imap_conn *imapc = &conn->proto.imapc; const char *line = data->state.buffer; - size_t wordlen; (void)instate; /* no use for this yet */ @@ -864,6 +863,7 @@ static CURLcode imap_state_capability_resp(struct connectdata *conn, /* Loop through the data line */ for(;;) { + size_t wordlen; while(*line && (*line == ' ' || *line == '\t' || *line == '\r' || *line == '\n')) { @@ -1046,12 +1046,12 @@ static CURLcode imap_state_select_resp(struct connectdata *conn, int imapcode, struct IMAP *imap = conn->data->req.protop; struct imap_conn *imapc = &conn->proto.imapc; const char *line = data->state.buffer; - char tmp[20]; (void)instate; /* no use for this yet */ if(imapcode == '*') { /* See if this is an UIDVALIDITY response */ + char tmp[20]; if(sscanf(line + 2, "OK [UIDVALIDITY %19[0123456789]]", tmp) == 1) { Curl_safefree(imapc->mailbox_uidvalidity); imapc->mailbox_uidvalidity = strdup(tmp); @@ -1119,7 +1119,7 @@ static CURLcode imap_state_fetch_resp(struct connectdata *conn, int imapcode, } if(parsed) { - infof(data, "Found %" CURL_FORMAT_CURL_OFF_TU " bytes to download\n", + infof(data, "Found %" CURL_FORMAT_CURL_OFF_T " bytes to download\n", size); Curl_pgrsSetDownloadSize(data, size); @@ -1144,10 +1144,8 @@ static CURLcode imap_state_fetch_resp(struct connectdata *conn, int imapcode, data->req.bytecount += chunk; - infof(data, "Written %" CURL_FORMAT_CURL_OFF_TU - " bytes, %" CURL_FORMAT_CURL_OFF_TU - " bytes are left for transfer\n", (curl_off_t)chunk, - size - chunk); + infof(data, "Written %zu bytes, %" CURL_FORMAT_CURL_OFF_TU + " bytes are left for transfer\n", chunk, size - chunk); /* Have we used the entire cache or just part of it?*/ if(pp->cache_size > chunk) { diff --git a/Utilities/cmcurl/lib/inet_ntop.c b/Utilities/cmcurl/lib/inet_ntop.c index fb91a50..ac5d2d4 100644 --- a/Utilities/cmcurl/lib/inet_ntop.c +++ b/Utilities/cmcurl/lib/inet_ntop.c @@ -49,7 +49,7 @@ */ static char *inet_ntop4 (const unsigned char *src, char *dst, size_t size) { - char tmp[sizeof "255.255.255.255"]; + char tmp[sizeof("255.255.255.255")]; size_t len; DEBUGASSERT(size >= 16); diff --git a/Utilities/cmcurl/lib/ldap.c b/Utilities/cmcurl/lib/ldap.c index 89047bc..4d8f4fa 100644 --- a/Utilities/cmcurl/lib/ldap.c +++ b/Utilities/cmcurl/lib/ldap.c @@ -54,15 +54,6 @@ # endif /* HAVE_LDAP_SSL && HAVE_LDAP_SSL_H */ #endif -/* These are macros in both <wincrypt.h> (in above <winldap.h>) and typedefs - * in BoringSSL's <openssl/x509.h> - */ -#ifdef HAVE_BORINGSSL -# undef X509_NAME -# undef X509_CERT_PAIR -# undef X509_EXTENSIONS -#endif - #include "urldata.h" #include <curl/curl.h> #include "sendf.h" diff --git a/Utilities/cmcurl/lib/md5.c b/Utilities/cmcurl/lib/md5.c index 3096602..b819d39 100644 --- a/Utilities/cmcurl/lib/md5.c +++ b/Utilities/cmcurl/lib/md5.c @@ -484,29 +484,35 @@ static void MD5_Final(unsigned char *result, MD5_CTX *ctx) #endif /* CRYPTO LIBS */ -/* Disable this picky gcc-8 compiler warning */ -#if defined(__GNUC__) && (__GNUC__ >= 8) -#pragma GCC diagnostic ignored "-Wcast-function-type" -#endif - const HMAC_params Curl_HMAC_MD5[] = { { - (HMAC_hinit_func) MD5_Init, /* Hash initialization function. */ - (HMAC_hupdate_func) MD5_Update, /* Hash update function. */ - (HMAC_hfinal_func) MD5_Final, /* Hash computation end function. */ - sizeof(MD5_CTX), /* Size of hash context structure. */ - 64, /* Maximum key length. */ - 16 /* Result size. */ + /* Hash initialization function. */ + CURLX_FUNCTION_CAST(HMAC_hinit_func, MD5_Init), + /* Hash update function. */ + CURLX_FUNCTION_CAST(HMAC_hupdate_func, MD5_Update), + /* Hash computation end function. */ + CURLX_FUNCTION_CAST(HMAC_hfinal_func, MD5_Final), + /* Size of hash context structure. */ + sizeof(MD5_CTX), + /* Maximum key length. */ + 64, + /* Result size. */ + 16 } }; const MD5_params Curl_DIGEST_MD5[] = { { - (Curl_MD5_init_func) MD5_Init, /* Digest initialization function */ - (Curl_MD5_update_func) MD5_Update, /* Digest update function */ - (Curl_MD5_final_func) MD5_Final, /* Digest computation end function */ - sizeof(MD5_CTX), /* Size of digest context struct */ - 16 /* Result size */ + /* Digest initialization function */ + CURLX_FUNCTION_CAST(Curl_MD5_init_func, MD5_Init), + /* Digest update function */ + CURLX_FUNCTION_CAST(Curl_MD5_update_func, MD5_Update), + /* Digest computation end function */ + CURLX_FUNCTION_CAST(Curl_MD5_final_func, MD5_Final), + /* Size of digest context struct */ + sizeof(MD5_CTX), + /* Result size */ + 16 } }; @@ -527,7 +533,7 @@ MD5_context *Curl_MD5_init(const MD5_params *md5params) MD5_context *ctxt; /* Create MD5 context */ - ctxt = malloc(sizeof *ctxt); + ctxt = malloc(sizeof(*ctxt)); if(!ctxt) return ctxt; diff --git a/Utilities/cmcurl/lib/memdebug.h b/Utilities/cmcurl/lib/memdebug.h index 6fb8b68..233de65 100644 --- a/Utilities/cmcurl/lib/memdebug.h +++ b/Utilities/cmcurl/lib/memdebug.h @@ -8,7 +8,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -137,13 +137,6 @@ CURL_EXTERN int curl_fclose(FILE *file, int line, const char *source); #endif #endif /* HAVE_GETADDRINFO */ -#ifdef HAVE_GETNAMEINFO -#undef getnameinfo -#define getnameinfo(sa,salen,host,hostlen,serv,servlen,flags) \ - curl_dogetnameinfo(sa, salen, host, hostlen, serv, servlen, flags, \ - __LINE__, __FILE__) -#endif /* HAVE_GETNAMEINFO */ - #ifdef HAVE_FREEADDRINFO #undef freeaddrinfo #define freeaddrinfo(data) \ diff --git a/Utilities/cmcurl/lib/mime.c b/Utilities/cmcurl/lib/mime.c index 4c0d2eeb..ca492d1 100644 --- a/Utilities/cmcurl/lib/mime.c +++ b/Utilities/cmcurl/lib/mime.c @@ -505,9 +505,6 @@ static size_t encoder_qp_read(char *buffer, size_t size, bool ateof, mime_encoder_state *st = &part->encstate; char *ptr = buffer; size_t cursize = 0; - int i; - size_t len; - size_t consumed; int softlinebreak; char buf[4]; @@ -516,9 +513,9 @@ static size_t encoder_qp_read(char *buffer, size_t size, bool ateof, character constants that can be interpreted as non-ascii on some platforms. Preserve ASCII encoding on output too. */ while(st->bufbeg < st->bufend) { - len = 1; - consumed = 1; - i = st->buf[st->bufbeg]; + size_t len = 1; + size_t consumed = 1; + int i = st->buf[st->bufbeg]; buf[0] = (char) i; buf[1] = aschex[(i >> 4) & 0xF]; buf[2] = aschex[i & 0xF]; @@ -787,10 +784,10 @@ static size_t read_encoded_part_content(curl_mimepart *part, st->bufbeg = 0; st->bufend = len; } - if(st->bufend >= sizeof st->buf) + if(st->bufend >= sizeof(st->buf)) return cursize? cursize: READ_ERROR; /* Buffer full. */ sz = read_part_content(part, st->buf + st->bufend, - sizeof st->buf - st->bufend); + sizeof(st->buf) - st->bufend); switch(sz) { case 0: ateof = TRUE; @@ -813,8 +810,6 @@ static size_t readback_part(curl_mimepart *part, char *buffer, size_t bufsize) { size_t cursize = 0; - size_t sz; - struct curl_slist *hdr; #ifdef CURL_DOES_CONVERSIONS char *convbuf = buffer; #endif @@ -822,8 +817,8 @@ static size_t readback_part(curl_mimepart *part, /* Readback from part. */ while(bufsize) { - sz = 0; - hdr = (struct curl_slist *) part->state.ptr; + size_t sz = 0; + struct curl_slist *hdr = (struct curl_slist *) part->state.ptr; switch(part->state.state) { case MIMESTATE_BEGIN: mimesetstate(&part->state, part->flags & MIME_BODY_ONLY? MIMESTATE_BODY: @@ -918,8 +913,6 @@ static size_t mime_subparts_read(char *buffer, size_t size, size_t nitems, { curl_mime *mime = (curl_mime *) instream; size_t cursize = 0; - size_t sz; - curl_mimepart *part; #ifdef CURL_DOES_CONVERSIONS char *convbuf = buffer; #endif @@ -927,8 +920,8 @@ static size_t mime_subparts_read(char *buffer, size_t size, size_t nitems, (void) size; /* Always 1. */ while(nitems) { - sz = 0; - part = mime->state.ptr; + size_t sz = 0; + curl_mimepart *part = mime->state.ptr; switch(mime->state.state) { case MIMESTATE_BEGIN: case MIMESTATE_BODY: @@ -1044,7 +1037,6 @@ static int mime_subparts_seek(void *instream, curl_off_t offset, int whence) curl_mime *mime = (curl_mime *) instream; curl_mimepart *part; int result = CURL_SEEKFUNC_OK; - int res; if(whence != SEEK_SET || offset) return CURL_SEEKFUNC_CANTSEEK; /* Only support full rewind. */ @@ -1053,7 +1045,7 @@ static int mime_subparts_seek(void *instream, curl_off_t offset, int whence) return CURL_SEEKFUNC_OK; /* Already rewound. */ for(part = mime->firstpart; part; part = part->nextpart) { - res = mime_part_rewind(part); + int res = mime_part_rewind(part); if(res != CURL_SEEKFUNC_OK) result = res; } @@ -1220,7 +1212,7 @@ curl_mime *curl_mime_init(struct Curl_easy *easy) { curl_mime *mime; - mime = (curl_mime *) malloc(sizeof *mime); + mime = (curl_mime *) malloc(sizeof(*mime)); if(mime) { mime->easy = easy; @@ -1236,8 +1228,13 @@ curl_mime *curl_mime_init(struct Curl_easy *easy) } memset(mime->boundary, '-', 24); - Curl_rand_hex(easy, (unsigned char *) mime->boundary + 24, - MIME_RAND_BOUNDARY_CHARS + 1); + if(Curl_rand_hex(easy, (unsigned char *) mime->boundary + 24, + MIME_RAND_BOUNDARY_CHARS + 1)) { + /* failed to get random separator, bail out */ + free(mime->boundary); + free(mime); + return NULL; + } mimesetstate(&mime->state, MIMESTATE_BEGIN, NULL); } @@ -1247,7 +1244,7 @@ curl_mime *curl_mime_init(struct Curl_easy *easy) /* Initialize a mime part. */ void Curl_mime_initpart(curl_mimepart *part, struct Curl_easy *easy) { - memset((char *) part, 0, sizeof *part); + memset((char *) part, 0, sizeof(*part)); part->easy = easy; mimesetstate(&part->state, MIMESTATE_BEGIN, NULL); } @@ -1260,7 +1257,7 @@ curl_mimepart *curl_mime_addpart(curl_mime *mime) if(!mime) return NULL; - part = (curl_mimepart *) malloc(sizeof *part); + part = (curl_mimepart *) malloc(sizeof(*part)); if(part) { Curl_mime_initpart(part, mime->easy); @@ -1349,7 +1346,6 @@ CURLcode curl_mime_data(curl_mimepart *part, CURLcode curl_mime_filedata(curl_mimepart *part, const char *filename) { CURLcode result = CURLE_OK; - char *base; if(!part) return CURLE_BAD_FUNCTION_ARGUMENT; @@ -1357,6 +1353,7 @@ CURLcode curl_mime_filedata(curl_mimepart *part, const char *filename) cleanup_part_content(part); if(filename) { + char *base; struct_stat sbuf; if(stat(filename, &sbuf) || access(filename, R_OK)) @@ -1564,7 +1561,6 @@ static size_t slist_size(struct curl_slist *s, static curl_off_t multipart_size(curl_mime *mime) { curl_off_t size; - curl_off_t sz; size_t boundarysize; curl_mimepart *part; @@ -1575,7 +1571,7 @@ static curl_off_t multipart_size(curl_mime *mime) size = boundarysize; /* Final boundary - CRLF after headers. */ for(part = mime->firstpart; part; part = part->nextpart) { - sz = Curl_mime_size(part); + curl_off_t sz = Curl_mime_size(part); if(sz < 0) size = sz; @@ -1643,8 +1639,6 @@ static CURLcode add_content_type(struct curl_slist **slp, const char *Curl_mime_contenttype(const char *filename) { - unsigned int i; - /* * If no content type was specified, we scan through a few well-known * extensions and pick the first we match! @@ -1669,8 +1663,9 @@ const char *Curl_mime_contenttype(const char *filename) if(filename) { size_t len1 = strlen(filename); const char *nameend = filename + len1; + unsigned int i; - for(i = 0; i < sizeof ctts / sizeof ctts[0]; i++) { + for(i = 0; i < sizeof(ctts) / sizeof(ctts[0]); i++) { size_t len2 = strlen(ctts[i].extension); if(len1 >= len2 && strcasecompare(nameend - len2, ctts[i].extension)) diff --git a/Utilities/cmcurl/lib/multi.c b/Utilities/cmcurl/lib/multi.c index f852846..0caf943 100644 --- a/Utilities/cmcurl/lib/multi.c +++ b/Utilities/cmcurl/lib/multi.c @@ -31,6 +31,7 @@ #include "progress.h" #include "easyif.h" #include "share.h" +#include "psl.h" #include "multiif.h" #include "sendf.h" #include "timeval.h" @@ -68,8 +69,8 @@ #define GOOD_MULTI_HANDLE(x) \ ((x) && (x)->type == CURL_MULTI_HANDLE) -static void singlesocket(struct Curl_multi *multi, - struct Curl_easy *data); +static CURLMcode singlesocket(struct Curl_multi *multi, + struct Curl_easy *data); static int update_timer(struct Curl_multi *multi); static CURLMcode add_next_timeout(struct curltime now, @@ -106,6 +107,16 @@ static const char * const statename[]={ /* function pointer called once when switching TO a state */ typedef void (*init_multistate_func)(struct Curl_easy *data); +static void Curl_init_completed(struct Curl_easy *data) +{ + /* this is a completed transfer */ + + /* Important: reset the conn pointer so that we don't point to memory + that could be freed anytime */ + data->easy_conn = NULL; + Curl_expire_clear(data); /* stop all timers */ +} + /* always use this function to change state, to make debugging easier */ static void mstate(struct Curl_easy *data, CURLMstate state #ifdef DEBUGBUILD @@ -115,17 +126,25 @@ static void mstate(struct Curl_easy *data, CURLMstate state { CURLMstate oldstate = data->mstate; static const init_multistate_func finit[CURLM_STATE_LAST] = { - NULL, - NULL, + NULL, /* INIT */ + NULL, /* CONNECT_PEND */ Curl_init_CONNECT, /* CONNECT */ - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - Curl_connect_free /* DO */ - /* the rest is NULL too */ + NULL, /* WAITRESOLVE */ + NULL, /* WAITCONNECT */ + NULL, /* WAITPROXYCONNECT */ + NULL, /* SENDPROTOCONNECT */ + NULL, /* PROTOCONNECT */ + NULL, /* WAITDO */ + Curl_connect_free, /* DO */ + NULL, /* DOING */ + NULL, /* DO_MORE */ + NULL, /* DO_DONE */ + NULL, /* WAITPERFORM */ + NULL, /* PERFORM */ + NULL, /* TOOFAST */ + NULL, /* DONE */ + Curl_init_completed, /* COMPLETED */ + NULL /* MSGSENT */ }; #if defined(DEBUGBUILD) && defined(CURL_DISABLE_VERBOSE_STRINGS) @@ -409,6 +428,14 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi, else data->state.conn_cache = &multi->conn_cache; +#ifdef USE_LIBPSL + /* Do the same for PSL. */ + if(data->share && (data->share->specifier & (1 << CURL_LOCK_DATA_PSL))) + data->psl = &data->share->psl; + else + data->psl = &multi->psl; +#endif + /* This adds the new entry at the 'end' of the doubly-linked circular list of Curl_easy structs to try and maintain a FIFO queue so the pipelined requests are in order. */ @@ -449,7 +476,7 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi, happens if the lastcall time is set to the same time when the handle is removed as when the next handle is added, as then the check in update_timer() that prevents calling the application multiple times with - the same timer infor will not trigger and then the new handle's timeout + the same timer info will not trigger and then the new handle's timeout will not be notified to the app. The work-around is thus simply to clear the 'lastcall' variable to force @@ -506,6 +533,11 @@ static CURLcode multi_done(struct connectdata **connp, /* Stop if multi_done() has already been called */ return CURLE_OK; + if(data->mstate == CURLM_STATE_WAITRESOLVE) { + /* still waiting for the resolve to complete */ + (void)Curl_resolver_wait_resolv(conn, NULL); + } + Curl_getoff_all_pipelines(data, conn); /* Cleanup possible redirect junk */ @@ -546,7 +578,7 @@ static CURLcode multi_done(struct connectdata **connp, if(conn->send_pipe.size || conn->recv_pipe.size) { /* Stop if pipeline is not empty . */ data->easy_conn = NULL; - DEBUGF(infof(data, "Connection still in use %d/%d, " + DEBUGF(infof(data, "Connection still in use %zu/%zu, " "no more multi_done now!\n", conn->send_pipe.size, conn->recv_pipe.size)); return CURLE_OK; @@ -560,6 +592,7 @@ static CURLcode multi_done(struct connectdata **connp, conn->dns_entry = NULL; } Curl_hostcache_prune(data); + Curl_safefree(data->state.ulbuf); /* if the transfer was completed in a paused state there can be buffered data left to free */ @@ -590,7 +623,7 @@ static CURLcode multi_done(struct connectdata **connp, #endif ) || conn->bits.close || (premature && !(conn->handler->flags & PROTOPT_STREAM))) { - CURLcode res2 = Curl_disconnect(conn, premature); /* close connection */ + CURLcode res2 = Curl_disconnect(data, conn, premature); /* If we had an error already, make sure we return that one. But if we got a new error, return that. */ @@ -608,7 +641,7 @@ static CURLcode multi_done(struct connectdata **connp, conn->bits.conn_to_host ? conn->conn_to_host.dispname : conn->host.dispname); - /* the connection is no longer in use */ + /* the connection is no longer in use by this transfer */ if(Curl_conncache_return_conn(conn)) { /* remember the most recently used connection */ data->state.lastconnect = conn; @@ -698,6 +731,11 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi, Curl_getoff_all_pipelines(data, data->easy_conn); } + if(data->connect_queue.ptr) + /* the handle was in the pending list waiting for an available connection, + so go ahead and remove it */ + Curl_llist_remove(&multi->pending, &data->connect_queue, NULL); + if(data->dns.hostcachetype == HCACHE_MULTI) { /* stop using the multi handle's DNS cache, *after* the possible multi_done() call above */ @@ -727,6 +765,12 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi, data->easy_conn = NULL; } +#ifdef USE_LIBPSL + /* Remove the PSL association. */ + if(data->psl == &multi->psl) + data->psl = NULL; +#endif + data->multi = NULL; /* clear the association to this multi handle */ /* make sure there's no pending message in the queue sent from this easy @@ -831,12 +875,10 @@ static int multi_getsock(struct Curl_easy *data, of sockets */ int numsocks) { - /* If the pipe broke, or if there's no connection left for this easy handle, - then we MUST bail out now with no bitmask set. The no connection case can - happen when this is called from curl_multi_remove_handle() => - singlesocket() => multi_getsock(). + /* The no connection case can happen when this is called from + curl_multi_remove_handle() => singlesocket() => multi_getsock(). */ - if(data->state.pipe_broke || !data->easy_conn) + if(!data->easy_conn) return 0; if(data->mstate > CURLM_STATE_CONNECT && @@ -902,7 +944,6 @@ CURLMcode curl_multi_fdset(struct Curl_multi *multi, struct Curl_easy *data; int this_max_fd = -1; curl_socket_t sockbunch[MAX_SOCKSPEREASYHANDLE]; - int bitmap; int i; (void)exc_fd_set; /* not used */ @@ -914,7 +955,7 @@ CURLMcode curl_multi_fdset(struct Curl_multi *multi, data = multi->easyp; while(data) { - bitmap = multi_getsock(data, sockbunch, MAX_SOCKSPEREASYHANDLE); + int bitmap = multi_getsock(data, sockbunch, MAX_SOCKSPEREASYHANDLE); for(i = 0; i< MAX_SOCKSPEREASYHANDLE; i++) { curl_socket_t s = CURL_SOCKET_BAD; @@ -1068,8 +1109,6 @@ CURLMcode curl_multi_wait(struct Curl_multi *multi, int pollrc; /* wait... */ pollrc = Curl_poll(ufds, nfds, timeout_ms); - DEBUGF(infof(data, "Curl_poll(%d ds, %d ms) == %d\n", - nfds, timeout_ms, pollrc)); if(pollrc > 0) { retcode = pollrc; @@ -1312,24 +1351,6 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, bool stream_error = FALSE; rc = CURLM_OK; - /* Handle the case when the pipe breaks, i.e., the connection - we're using gets cleaned up and we're left with nothing. */ - if(data->state.pipe_broke) { - infof(data, "Pipe broke: handle %p, url = %s\n", - (void *)data, data->state.path); - - if(data->mstate < CURLM_STATE_COMPLETED) { - /* Head back to the CONNECT state */ - multistate(data, CURLM_STATE_CONNECT); - rc = CURLM_CALL_MULTI_PERFORM; - result = CURLE_OK; - } - - data->state.pipe_broke = FALSE; - data->easy_conn = NULL; - continue; - } - if(!data->easy_conn && data->mstate > CURLM_STATE_CONNECT && data->mstate < CURLM_STATE_DONE) { @@ -1555,6 +1576,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, multistate(data, CURLM_STATE_SENDPROTOCONNECT); } } + else if(result) + stream_error = TRUE; break; #endif @@ -1909,6 +1932,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, CURLcode ret = Curl_retry_request(data->easy_conn, &newurl); if(!ret) retry = (newurl)?TRUE:FALSE; + else if(!result) + result = ret; if(retry) { /* if we are to retry, set the result to OK and consider the @@ -2040,16 +2065,6 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, break; case CURLM_STATE_COMPLETED: - /* this is a completed transfer, it is likely to still be connected */ - - /* This node should be delinked from the list now and we should post - an information message that we are complete. */ - - /* Important: reset the conn pointer so that we don't point to memory - that could be freed anytime */ - data->easy_conn = NULL; - - Curl_expire_clear(data); /* stop all timers */ break; case CURLM_STATE_MSGSENT: @@ -2071,8 +2086,6 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, /* NOTE: no attempt to disconnect connections must be made in the case blocks above - cleanup happens only here */ - data->state.pipe_broke = FALSE; - /* Check if we can move pending requests to send pipe */ process_pending_handles(multi); /* connection */ @@ -2087,7 +2100,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, /* Don't attempt to send data over a connection that timed out */ bool dead_connection = result == CURLE_OPERATION_TIMEDOUT; /* disconnect properly */ - Curl_disconnect(data->easy_conn, dead_connection); + Curl_disconnect(data, data->easy_conn, dead_connection); /* This is where we make sure that the easy_conn pointer is reset. We don't have to do this in every case block above where a @@ -2101,6 +2114,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, } multistate(data, CURLM_STATE_COMPLETED); + rc = CURLM_CALL_MULTI_PERFORM; } /* if there's still a connection to use, call the progress function */ else if(data->easy_conn && Curl_pgrsUpdate(data->easy_conn)) { @@ -2125,14 +2139,12 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, msg->extmsg.data.result = result; rc = multi_addmsg(multi, msg); - + DEBUGASSERT(!data->easy_conn); multistate(data, CURLM_STATE_MSGSENT); } } while((rc == CURLM_CALL_MULTI_PERFORM) || multi_ischanged(multi, FALSE)); data->result = result; - - return rc; } @@ -2220,6 +2232,11 @@ CURLMcode curl_multi_cleanup(struct Curl_multi *multi) data->state.conn_cache = NULL; data->multi = NULL; /* clear the association */ +#ifdef USE_LIBPSL + if(data->psl == &multi->psl) + data->psl = NULL; +#endif + data = nextdata; } @@ -2232,6 +2249,7 @@ CURLMcode curl_multi_cleanup(struct Curl_multi *multi) Curl_llist_destroy(&multi->pending, NULL); Curl_hash_destroy(&multi->hostcache); + Curl_psl_destroy(&multi->psl); /* Free the blacklists by setting them to NULL */ Curl_pipeline_set_site_blacklist(NULL, &multi->pipelining_site_bl); @@ -2286,8 +2304,8 @@ CURLMsg *curl_multi_info_read(struct Curl_multi *multi, int *msgs_in_queue) * and if we have a different state in any of those sockets from last time we * call the callback accordingly. */ -static void singlesocket(struct Curl_multi *multi, - struct Curl_easy *data) +static CURLMcode singlesocket(struct Curl_multi *multi, + struct Curl_easy *data) { curl_socket_t socks[MAX_SOCKSPEREASYHANDLE]; int i; @@ -2334,7 +2352,7 @@ static void singlesocket(struct Curl_multi *multi, entry = sh_addentry(&multi->sockhash, s, data); if(!entry) /* fatal */ - return; + return CURLM_OUT_OF_MEMORY; } /* we know (entry != NULL) at this point, see the logic above */ @@ -2422,6 +2440,7 @@ static void singlesocket(struct Curl_multi *multi, memcpy(data->sockets, socks, num*sizeof(curl_socket_t)); data->numsocks = num; + return CURLM_OK; } void Curl_updatesocket(struct Curl_easy *data) @@ -2442,20 +2461,23 @@ void Curl_updatesocket(struct Curl_easy *data) void Curl_multi_closed(struct connectdata *conn, curl_socket_t s) { - struct Curl_multi *multi = conn->data->multi; - if(multi) { - /* this is set if this connection is part of a handle that is added to - a multi handle, and only then this is necessary */ - struct Curl_sh_entry *entry = sh_getentry(&multi->sockhash, s); - - if(entry) { - if(multi->socket_cb) - multi->socket_cb(conn->data, s, CURL_POLL_REMOVE, - multi->socket_userp, - entry->socketp); + if(conn->data) { + /* if there's still an easy handle associated with this connection */ + struct Curl_multi *multi = conn->data->multi; + if(multi) { + /* this is set if this connection is part of a handle that is added to + a multi handle, and only then this is necessary */ + struct Curl_sh_entry *entry = sh_getentry(&multi->sockhash, s); + + if(entry) { + if(multi->socket_cb) + multi->socket_cb(conn->data, s, CURL_POLL_REMOVE, + multi->socket_userp, + entry->socketp); - /* now remove it from the socket hash */ - sh_delentry(&multi->sockhash, s); + /* now remove it from the socket hash */ + sh_delentry(&multi->sockhash, s); + } } } } @@ -2535,8 +2557,8 @@ static CURLMcode multi_socket(struct Curl_multi *multi, and callbacks */ if(result != CURLM_BAD_HANDLE) { data = multi->easyp; - while(data) { - singlesocket(multi, data); + while(data && !result) { + result = singlesocket(multi, data); data = data->next; } } @@ -2590,10 +2612,13 @@ static CURLMcode multi_socket(struct Curl_multi *multi, /* clear the bitmask only if not locked */ data->easy_conn->cselect_bits = 0; - if(CURLM_OK >= result) + if(CURLM_OK >= result) { /* get the socket(s) and check if the state has been changed since last */ - singlesocket(multi, data); + result = singlesocket(multi, data); + if(result) + return result; + } /* Now we fall-through and do the timer-based stuff, since we don't want to force the user to have to deal with timeouts as long as at least @@ -2627,10 +2652,13 @@ static CURLMcode multi_socket(struct Curl_multi *multi, result = multi_runsingle(multi, now, data); sigpipe_restore(&pipe_st); - if(CURLM_OK >= result) + if(CURLM_OK >= result) { /* get the socket(s) and check if the state has been changed since last */ - singlesocket(multi, data); + result = singlesocket(multi, data); + if(result) + return result; + } } /* Check if there's one (more) expired timer to deal with! This function @@ -2924,7 +2952,6 @@ void Curl_expire(struct Curl_easy *data, time_t milli, expire_id id) { struct Curl_multi *multi = data->multi; struct curltime *nowp = &data->state.expiretime; - int rc; struct curltime set; /* this is only interesting while there is still an associated multi struct @@ -2955,6 +2982,7 @@ void Curl_expire(struct Curl_easy *data, time_t milli, expire_id id) Compare if the new time is earlier, and only remove-old/add-new if it is. */ timediff_t diff = Curl_timediff(set, *nowp); + int rc; if(diff > 0) { /* The current splay tree entry is sooner than this new expiry time. @@ -3000,7 +3028,6 @@ void Curl_expire_clear(struct Curl_easy *data) { struct Curl_multi *multi = data->multi; struct curltime *nowp = &data->state.expiretime; - int rc; /* this is only interesting while there is still an associated multi struct remaining! */ @@ -3011,6 +3038,7 @@ void Curl_expire_clear(struct Curl_easy *data) /* Since this is an cleared time, we must remove the previous entry from the splay tree */ struct curl_llist *list = &data->state.timeoutlist; + int rc; rc = Curl_splayremovebyaddr(multi->timetree, &data->state.timenode, @@ -3100,12 +3128,15 @@ static void process_pending_handles(struct Curl_multi *multi) } } -void Curl_set_in_callback(struct Curl_easy *easy, bool value) +void Curl_set_in_callback(struct Curl_easy *data, bool value) { - if(easy->multi_easy) - easy->multi_easy->in_callback = value; - else if(easy->multi) - easy->multi->in_callback = value; + /* might get called when there is no data pointer! */ + if(data) { + if(data->multi_easy) + data->multi_easy->in_callback = value; + else if(data->multi) + data->multi->in_callback = value; + } } bool Curl_is_in_callback(struct Curl_easy *easy) diff --git a/Utilities/cmcurl/lib/multihandle.h b/Utilities/cmcurl/lib/multihandle.h index 1a5017f..ea2bf35 100644 --- a/Utilities/cmcurl/lib/multihandle.h +++ b/Utilities/cmcurl/lib/multihandle.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -23,6 +23,7 @@ ***************************************************************************/ #include "conncache.h" +#include "psl.h" struct Curl_message { struct curl_llist_element list; @@ -97,6 +98,11 @@ struct Curl_multi { /* Hostname cache */ struct curl_hash hostcache; +#ifdef USE_LIBPSL + /* PSL cache. */ + struct PslCache psl; +#endif + /* timetree points to the splay-tree of time nodes to figure out expire times of all currently set timers */ struct Curl_tree *timetree; diff --git a/Utilities/cmcurl/lib/netrc.c b/Utilities/cmcurl/lib/netrc.c index dbcc59a..a407bda 100644 --- a/Utilities/cmcurl/lib/netrc.c +++ b/Utilities/cmcurl/lib/netrc.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -115,7 +115,7 @@ int Curl_parsenetrc(const char *host, char *tok; char *tok_buf; bool done = FALSE; - char netrcbuffer[256]; + char netrcbuffer[4096]; int netrcbuffsize = (int)sizeof(netrcbuffer); while(!done && fgets(netrcbuffer, netrcbuffsize, file)) { diff --git a/Utilities/cmcurl/lib/pingpong.c b/Utilities/cmcurl/lib/pingpong.c index ad370ee..2e93d20 100644 --- a/Utilities/cmcurl/lib/pingpong.c +++ b/Utilities/cmcurl/lib/pingpong.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -49,7 +49,6 @@ time_t Curl_pp_state_timeout(struct pingpong *pp) struct connectdata *conn = pp->conn; struct Curl_easy *data = conn->data; time_t timeout_ms; /* in milliseconds */ - time_t timeout2_ms; /* in milliseconds */ long response_time = (data->set.server_response_timeout)? data->set.server_response_timeout: pp->response_time; @@ -65,7 +64,7 @@ time_t Curl_pp_state_timeout(struct pingpong *pp) if(data->set.timeout) { /* if timeout is requested, find out how much remaining time we have */ - timeout2_ms = data->set.timeout - /* timeout time */ + time_t timeout2_ms = data->set.timeout - /* timeout time */ Curl_timediff(Curl_now(), conn->now); /* spent time */ /* pick the lowest number */ @@ -222,8 +221,7 @@ CURLcode Curl_pp_vsendf(struct pingpong *pp, } if(conn->data->set.verbose) - Curl_debug(conn->data, CURLINFO_HEADER_OUT, - s, (size_t)bytes_written, conn); + Curl_debug(conn->data, CURLINFO_HEADER_OUT, s, (size_t)bytes_written); if(bytes_written != (ssize_t)write_len) { /* the whole chunk was not sent, keep it around and adjust sizes */ @@ -371,7 +369,7 @@ CURLcode Curl_pp_readresp(curl_socket_t sockfd, #endif if(data->set.verbose) Curl_debug(data, CURLINFO_HEADER_IN, - pp->linestart_resp, (size_t)perline, conn); + pp->linestart_resp, (size_t)perline); /* * We pass all response-lines to the callback function registered diff --git a/Utilities/cmcurl/lib/pipeline.c b/Utilities/cmcurl/lib/pipeline.c index 4d41b04..8de3bab 100644 --- a/Utilities/cmcurl/lib/pipeline.c +++ b/Utilities/cmcurl/lib/pipeline.c @@ -6,7 +6,7 @@ * \___|\___/|_| \_\_____| * * Copyright (C) 2013, Linus Nielsen Feltzing, <linus@haxx.se> - * Copyright (C) 2013 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 2013 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -82,7 +82,8 @@ bool Curl_pipeline_penalized(struct Curl_easy *data, penalized = TRUE; infof(data, "Conn: %ld (%p) Receive pipe weight: (%" - CURL_FORMAT_CURL_OFF_T "/%zu), penalized: %s\n", + CURL_FORMAT_CURL_OFF_T "/%" CURL_FORMAT_CURL_OFF_T + "), penalized: %s\n", conn->connection_id, (void *)conn, recv_size, conn->chunk.datasize, penalized?"TRUE":"FALSE"); return penalized; @@ -109,8 +110,8 @@ CURLcode Curl_add_handle_to_pipeline(struct Curl_easy *handle, pipeline = &conn->send_pipe; result = addHandleToPipeline(handle, pipeline); - - if(pipeline == &conn->send_pipe && sendhead != conn->send_pipe.head) { + if((conn->bundle->multiuse == BUNDLE_PIPELINING) && + (pipeline == &conn->send_pipe && sendhead != conn->send_pipe.head)) { /* this is a new one as head, expire it */ Curl_pipeline_leave_write(conn); /* not in use yet */ Curl_expire(conn->send_pipe.head->ptr, 0, EXPIRE_RUN_NOW); diff --git a/Utilities/cmcurl/lib/pop3.c b/Utilities/cmcurl/lib/pop3.c index 78f6afe..cd994f6 100644 --- a/Utilities/cmcurl/lib/pop3.c +++ b/Utilities/cmcurl/lib/pop3.c @@ -613,7 +613,6 @@ static CURLcode pop3_state_servergreet_resp(struct connectdata *conn, struct pop3_conn *pop3c = &conn->proto.pop3c; const char *line = data->state.buffer; size_t len = strlen(line); - size_t i; (void)instate; /* no use for this yet */ @@ -625,6 +624,7 @@ static CURLcode pop3_state_servergreet_resp(struct connectdata *conn, /* Does the server support APOP authentication? */ if(len >= 4 && line[len - 2] == '>') { /* Look for the APOP timestamp */ + size_t i; for(i = 3; i < len - 2; ++i) { if(line[i] == '<') { /* Calculate the length of the timestamp */ @@ -664,7 +664,6 @@ static CURLcode pop3_state_capa_resp(struct connectdata *conn, int pop3code, struct pop3_conn *pop3c = &conn->proto.pop3c; const char *line = data->state.buffer; size_t len = strlen(line); - size_t wordlen; (void)instate; /* no use for this yet */ @@ -689,6 +688,7 @@ static CURLcode pop3_state_capa_resp(struct connectdata *conn, int pop3code, /* Loop through the data line */ for(;;) { size_t llen; + size_t wordlen; unsigned int mechbit; while(len && diff --git a/Utilities/cmcurl/lib/progress.c b/Utilities/cmcurl/lib/progress.c index f59faa3..a94668d 100644 --- a/Utilities/cmcurl/lib/progress.c +++ b/Utilities/cmcurl/lib/progress.c @@ -35,22 +35,22 @@ byte) */ static void time2str(char *r, curl_off_t seconds) { - curl_off_t d, h, m, s; + curl_off_t h; if(seconds <= 0) { strcpy(r, "--:--:--"); return; } h = seconds / CURL_OFF_T_C(3600); if(h <= CURL_OFF_T_C(99)) { - m = (seconds - (h*CURL_OFF_T_C(3600))) / CURL_OFF_T_C(60); - s = (seconds - (h*CURL_OFF_T_C(3600))) - (m*CURL_OFF_T_C(60)); + curl_off_t m = (seconds - (h*CURL_OFF_T_C(3600))) / CURL_OFF_T_C(60); + curl_off_t s = (seconds - (h*CURL_OFF_T_C(3600))) - (m*CURL_OFF_T_C(60)); snprintf(r, 9, "%2" CURL_FORMAT_CURL_OFF_T ":%02" CURL_FORMAT_CURL_OFF_T ":%02" CURL_FORMAT_CURL_OFF_T, h, m, s); } else { /* this equals to more than 99 hours, switch to a more suitable output format to fit within the limits. */ - d = seconds / CURL_OFF_T_C(86400); + curl_off_t d = seconds / CURL_OFF_T_C(86400); h = (seconds - (d*CURL_OFF_T_C(86400))) / CURL_OFF_T_C(3600); if(d <= CURL_OFF_T_C(999)) snprintf(r, 9, "%3" CURL_FORMAT_CURL_OFF_T @@ -369,25 +369,10 @@ void Curl_pgrsSetUploadSize(struct Curl_easy *data, curl_off_t size) int Curl_pgrsUpdate(struct connectdata *conn) { struct curltime now; - int result; - char max5[6][10]; - curl_off_t dlpercen = 0; - curl_off_t ulpercen = 0; - curl_off_t total_percen = 0; - curl_off_t total_transfer; - curl_off_t total_expected_transfer; curl_off_t timespent; curl_off_t timespent_ms; /* milliseconds */ struct Curl_easy *data = conn->data; int nowindex = data->progress.speeder_c% CURR_TIME; - int checkindex; - int countindex; /* amount of seconds stored in the speeder array */ - char time_left[10]; - char time_total[10]; - char time_spent[10]; - curl_off_t ulestimate = 0; - curl_off_t dlestimate = 0; - curl_off_t total_estimate; bool shownow = FALSE; curl_off_t dl = data->progress.downloaded; curl_off_t ul = data->progress.uploaded; @@ -413,6 +398,7 @@ int Curl_pgrsUpdate(struct connectdata *conn) /* Calculations done at most once a second, unless end is reached */ if(data->progress.lastshow != now.tv_sec) { + int countindex; /* amount of seconds stored in the speeder array */ shownow = TRUE; data->progress.lastshow = now.tv_sec; @@ -438,6 +424,7 @@ int Curl_pgrsUpdate(struct connectdata *conn) /* first of all, we don't do this if there's no counted seconds yet */ if(countindex) { + int checkindex; timediff_t span_ms; /* Get the index position to compare with the 'nowindex' position. @@ -477,8 +464,21 @@ int Curl_pgrsUpdate(struct connectdata *conn) if(!(data->progress.flags & PGRS_HIDE)) { /* progress meter has not been shut off */ + char max5[6][10]; + curl_off_t dlpercen = 0; + curl_off_t ulpercen = 0; + curl_off_t total_percen = 0; + curl_off_t total_transfer; + curl_off_t total_expected_transfer; + char time_left[10]; + char time_total[10]; + char time_spent[10]; + curl_off_t ulestimate = 0; + curl_off_t dlestimate = 0; + curl_off_t total_estimate; if(data->set.fxferinfo) { + int result; /* There's a callback set, call that */ Curl_set_in_callback(data, true); result = data->set.fxferinfo(data->set.progress_client, @@ -492,6 +492,7 @@ int Curl_pgrsUpdate(struct connectdata *conn) return result; } if(data->set.fprogress) { + int result; /* The older deprecated callback is set, call that */ Curl_set_in_callback(data, true); result = data->set.fprogress(data->set.progress_client, diff --git a/Utilities/cmcurl/lib/progress.h b/Utilities/cmcurl/lib/progress.h index 2baa925..92dbcbd 100644 --- a/Utilities/cmcurl/lib/progress.h +++ b/Utilities/cmcurl/lib/progress.h @@ -56,23 +56,10 @@ timediff_t Curl_pgrsLimitWaitTime(curl_off_t cursize, struct curltime start, struct curltime now); -/* Don't show progress for sizes smaller than: */ -#define LEAST_SIZE_PROGRESS BUFSIZE - -#define PROGRESS_DOWNLOAD (1<<0) -#define PROGRESS_UPLOAD (1<<1) -#define PROGRESS_DOWN_AND_UP (PROGRESS_UPLOAD | PROGRESS_DOWNLOAD) - -#define PGRS_SHOW_DL (1<<0) -#define PGRS_SHOW_UL (1<<1) -#define PGRS_DONE_DL (1<<2) -#define PGRS_DONE_UL (1<<3) #define PGRS_HIDE (1<<4) #define PGRS_UL_SIZE_KNOWN (1<<5) #define PGRS_DL_SIZE_KNOWN (1<<6) - #define PGRS_HEADERS_OUT (1<<7) /* set when the headers have been written */ - #endif /* HEADER_CURL_PROGRESS_H */ diff --git a/Utilities/cmcurl/lib/psl.c b/Utilities/cmcurl/lib/psl.c new file mode 100644 index 0000000..568baff --- /dev/null +++ b/Utilities/cmcurl/lib/psl.c @@ -0,0 +1,111 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include "curl_setup.h" + +#include <curl/curl.h> + +#ifdef USE_LIBPSL + +#include "psl.h" +#include "share.h" + +/* The last 3 #include files should be in this order */ +#include "curl_printf.h" +#include "curl_memory.h" +#include "memdebug.h" + +void Curl_psl_destroy(struct PslCache *pslcache) +{ + if(pslcache->psl) { + if(pslcache->dynamic) + psl_free((psl_ctx_t *) pslcache->psl); + pslcache->psl = NULL; + pslcache->dynamic = FALSE; + } +} + +static time_t now_seconds(void) +{ + struct curltime now = Curl_now(); + + return now.tv_sec; +} + +const psl_ctx_t *Curl_psl_use(struct Curl_easy *easy) +{ + struct PslCache *pslcache = easy->psl; + const psl_ctx_t *psl; + time_t now; + + if(!pslcache) + return NULL; + + Curl_share_lock(easy, CURL_LOCK_DATA_PSL, CURL_LOCK_ACCESS_SHARED); + now = now_seconds(); + if(!pslcache->psl || pslcache->expires <= now) { + /* Let a chance to other threads to do the job: avoids deadlock. */ + Curl_share_unlock(easy, CURL_LOCK_DATA_PSL); + + /* Update cache: this needs an exclusive lock. */ + Curl_share_lock(easy, CURL_LOCK_DATA_PSL, CURL_LOCK_ACCESS_SINGLE); + + /* Recheck in case another thread did the job. */ + now = now_seconds(); + if(!pslcache->psl || pslcache->expires <= now) { + bool dynamic = FALSE; + time_t expires = TIME_T_MAX; + +#if defined(PSL_VERSION_NUMBER) && PSL_VERSION_NUMBER >= 0x001000 + psl = psl_latest(NULL); + dynamic = psl != NULL; + /* Take care of possible time computation overflow. */ + expires = now < TIME_T_MAX - PSL_TTL? now + PSL_TTL: TIME_T_MAX; + + /* Only get the built-in PSL if we do not already have the "latest". */ + if(!psl && !pslcache->dynamic) +#endif + + psl = psl_builtin(); + + if(psl) { + Curl_psl_destroy(pslcache); + pslcache->psl = psl; + pslcache->dynamic = dynamic; + pslcache->expires = expires; + } + } + Curl_share_unlock(easy, CURL_LOCK_DATA_PSL); /* Release exclusive lock. */ + Curl_share_lock(easy, CURL_LOCK_DATA_PSL, CURL_LOCK_ACCESS_SHARED); + } + psl = pslcache->psl; + if(!psl) + Curl_share_unlock(easy, CURL_LOCK_DATA_PSL); + return psl; +} + +void Curl_psl_release(struct Curl_easy *easy) +{ + Curl_share_unlock(easy, CURL_LOCK_DATA_PSL); +} + +#endif /* USE_LIBPSL */ diff --git a/Utilities/cmcurl/lib/psl.h b/Utilities/cmcurl/lib/psl.h new file mode 100644 index 0000000..e9f99d0 --- /dev/null +++ b/Utilities/cmcurl/lib/psl.h @@ -0,0 +1,47 @@ +#ifndef HEADER_PSL_H +#define HEADER_PSL_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#ifdef USE_LIBPSL +#include <libpsl.h> + +#define PSL_TTL (72 * 3600) /* PSL time to live before a refresh. */ + +struct PslCache { + const psl_ctx_t *psl; /* The PSL. */ + time_t expires; /* Time this PSL life expires. */ + bool dynamic; /* PSL should be released when no longer needed. */ +}; + +const psl_ctx_t *Curl_psl_use(struct Curl_easy *easy); +void Curl_psl_release(struct Curl_easy *easy); +void Curl_psl_destroy(struct PslCache *pslcache); + +#else + +#define Curl_psl_use(easy) NULL +#define Curl_psl_release(easy) +#define Curl_psl_destroy(pslcache) + +#endif /* USE_LIBPSL */ +#endif /* HEADER_PSL_H */ diff --git a/Utilities/cmcurl/lib/rand.c b/Utilities/cmcurl/lib/rand.c index 0769ed1..1dc2504 100644 --- a/Utilities/cmcurl/lib/rand.c +++ b/Utilities/cmcurl/lib/rand.c @@ -158,7 +158,7 @@ CURLcode Curl_rand_hex(struct Curl_easy *data, unsigned char *rnd, DEBUGASSERT(num > 1); #ifdef __clang_analyzer__ - /* This silences a scan-build warning about accesssing this buffer with + /* This silences a scan-build warning about accessing this buffer with uninitialized memory. */ memset(buffer, 0, sizeof(buffer)); #endif diff --git a/Utilities/cmcurl/lib/rtsp.c b/Utilities/cmcurl/lib/rtsp.c index 41f3000..182ee29 100644 --- a/Utilities/cmcurl/lib/rtsp.c +++ b/Utilities/cmcurl/lib/rtsp.c @@ -218,8 +218,6 @@ static CURLcode rtsp_done(struct connectdata *conn, struct Curl_easy *data = conn->data; struct RTSP *rtsp = data->req.protop; CURLcode httpStatus; - long CSeq_sent; - long CSeq_recv; /* Bypass HTTP empty-reply checks on receive */ if(data->set.rtspreq == RTSPREQ_RECEIVE) @@ -229,8 +227,8 @@ static CURLcode rtsp_done(struct connectdata *conn, if(rtsp) { /* Check the sequence numbers */ - CSeq_sent = rtsp->CSeq_sent; - CSeq_recv = rtsp->CSeq_recv; + long CSeq_sent = rtsp->CSeq_sent; + long CSeq_recv = rtsp->CSeq_recv; if((data->set.rtspreq != RTSPREQ_RECEIVE) && (CSeq_sent != CSeq_recv)) { failf(data, "The CSeq of this request %ld did not match the response %ld", diff --git a/Utilities/cmcurl/lib/security.c b/Utilities/cmcurl/lib/security.c index d171985..4034115 100644 --- a/Utilities/cmcurl/lib/security.c +++ b/Utilities/cmcurl/lib/security.c @@ -390,7 +390,7 @@ int Curl_sec_read_msg(struct connectdata *conn, char *buffer, if(conn->data->set.verbose) { buf[decoded_len] = '\n'; - Curl_debug(conn->data, CURLINFO_HEADER_IN, buf, decoded_len + 1, conn); + Curl_debug(conn->data, CURLINFO_HEADER_IN, buf, decoded_len + 1); } buf[decoded_len] = '\0'; @@ -488,7 +488,7 @@ static CURLcode choose_mech(struct connectdata *conn) tmp_allocation = realloc(conn->app_data, mech->size); if(tmp_allocation == NULL) { - failf(data, "Failed realloc of size %u", mech->size); + failf(data, "Failed realloc of size %zu", mech->size); mech = NULL; return CURLE_OUT_OF_MEMORY; } diff --git a/Utilities/cmcurl/lib/select.c b/Utilities/cmcurl/lib/select.c index 0406dd2..6e73890 100644 --- a/Utilities/cmcurl/lib/select.c +++ b/Utilities/cmcurl/lib/select.c @@ -80,7 +80,6 @@ int Curl_wait_ms(int timeout_ms) #endif struct curltime initial_tv; int pending_ms; - int error; #endif int r = 0; @@ -98,6 +97,7 @@ int Curl_wait_ms(int timeout_ms) pending_ms = timeout_ms; initial_tv = Curl_now(); do { + int error; #if defined(HAVE_POLL_FINE) r = poll(NULL, 0, pending_ms); #else @@ -160,7 +160,6 @@ int Curl_socket_check(curl_socket_t readfd0, /* two sockets to read from */ #endif struct curltime initial_tv = {0, 0}; int pending_ms = 0; - int error; int r; int ret; @@ -210,6 +209,7 @@ int Curl_socket_check(curl_socket_t readfd0, /* two sockets to read from */ } do { + int error; if(timeout_ms < 0) pending_ms = -1; else if(!timeout_ms) @@ -291,6 +291,7 @@ int Curl_socket_check(curl_socket_t readfd0, /* two sockets to read from */ ptimeout = (timeout_ms < 0) ? NULL : &pending_tv; do { + int error; if(timeout_ms > 0) { pending_tv.tv_sec = pending_ms / 1000; pending_tv.tv_usec = (pending_ms % 1000) * 1000; @@ -402,7 +403,6 @@ int Curl_poll(struct pollfd ufds[], unsigned int nfds, int timeout_ms) bool fds_none = TRUE; unsigned int i; int pending_ms = 0; - int error; int r; if(ufds) { @@ -431,6 +431,7 @@ int Curl_poll(struct pollfd ufds[], unsigned int nfds, int timeout_ms) #ifdef HAVE_POLL_FINE do { + int error; if(timeout_ms < 0) pending_ms = -1; else if(!timeout_ms) @@ -502,6 +503,7 @@ int Curl_poll(struct pollfd ufds[], unsigned int nfds, int timeout_ms) ptimeout = (timeout_ms < 0) ? NULL : &pending_tv; do { + int error; if(timeout_ms > 0) { pending_tv.tv_sec = pending_ms / 1000; pending_tv.tv_usec = (pending_ms % 1000) * 1000; diff --git a/Utilities/cmcurl/lib/sendf.c b/Utilities/cmcurl/lib/sendf.c index 27c0ccc..d3c10b3 100644 --- a/Utilities/cmcurl/lib/sendf.c +++ b/Utilities/cmcurl/lib/sendf.c @@ -240,7 +240,7 @@ void Curl_infof(struct Curl_easy *data, const char *fmt, ...) vsnprintf(print_buffer, sizeof(print_buffer), fmt, ap); va_end(ap); len = strlen(print_buffer); - Curl_debug(data, CURLINFO_TEXT, print_buffer, len, NULL); + Curl_debug(data, CURLINFO_TEXT, print_buffer, len); } } @@ -265,7 +265,7 @@ void Curl_failf(struct Curl_easy *data, const char *fmt, ...) if(data->set.verbose) { error[len] = '\n'; error[++len] = '\0'; - Curl_debug(data, CURLINFO_TEXT, error, len, NULL); + Curl_debug(data, CURLINFO_TEXT, error, len); } va_end(ap); } @@ -300,7 +300,7 @@ CURLcode Curl_sendf(curl_socket_t sockfd, struct connectdata *conn, break; if(data->set.verbose) - Curl_debug(data, CURLINFO_DATA_OUT, sptr, (size_t)bytes_written, conn); + Curl_debug(data, CURLINFO_DATA_OUT, sptr, (size_t)bytes_written); if((size_t)bytes_written != write_len) { /* if not all was written at once, we must advance the pointer, decrease @@ -762,8 +762,8 @@ CURLcode Curl_read(struct connectdata *conn, /* connection data */ } /* return 0 on success */ -static int showit(struct Curl_easy *data, curl_infotype type, - char *ptr, size_t size) +int Curl_debug(struct Curl_easy *data, curl_infotype type, + char *ptr, size_t size) { static const char s_infotype[CURLINFO_END][3] = { "* ", "< ", "> ", "{ ", "} ", "{ ", "} " }; @@ -834,41 +834,3 @@ static int showit(struct Curl_easy *data, curl_infotype type, #endif return rc; } - -int Curl_debug(struct Curl_easy *data, curl_infotype type, - char *ptr, size_t size, - struct connectdata *conn) -{ - int rc; - if(data->set.printhost && conn && conn->host.dispname) { - char buffer[160]; - const char *t = NULL; - const char *w = "Data"; - switch(type) { - case CURLINFO_HEADER_IN: - w = "Header"; - /* FALLTHROUGH */ - case CURLINFO_DATA_IN: - t = "from"; - break; - case CURLINFO_HEADER_OUT: - w = "Header"; - /* FALLTHROUGH */ - case CURLINFO_DATA_OUT: - t = "to"; - break; - default: - break; - } - - if(t) { - snprintf(buffer, sizeof(buffer), "[%s %s %s]", w, t, - conn->host.dispname); - rc = showit(data, CURLINFO_TEXT, buffer, strlen(buffer)); - if(rc) - return rc; - } - } - rc = showit(data, type, ptr, size); - return rc; -} diff --git a/Utilities/cmcurl/lib/sendf.h b/Utilities/cmcurl/lib/sendf.h index 7c9134d..7627fe6 100644 --- a/Utilities/cmcurl/lib/sendf.h +++ b/Utilities/cmcurl/lib/sendf.h @@ -84,9 +84,8 @@ CURLcode Curl_write_plain(struct connectdata *conn, ssize_t *written); /* the function used to output verbose information */ -int Curl_debug(struct Curl_easy *handle, curl_infotype type, - char *data, size_t size, - struct connectdata *conn); +int Curl_debug(struct Curl_easy *data, curl_infotype type, + char *ptr, size_t size); #endif /* HEADER_CURL_SENDF_H */ diff --git a/Utilities/cmcurl/lib/setopt.c b/Utilities/cmcurl/lib/setopt.c index af53ee3..5c5f4b3 100644 --- a/Utilities/cmcurl/lib/setopt.c +++ b/Utilities/cmcurl/lib/setopt.c @@ -142,6 +142,25 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_arg(param, char *)); break; + case CURLOPT_TLS13_CIPHERS: + if(Curl_ssl_tls13_ciphersuites()) { + /* set preferred list of TLS 1.3 cipher suites */ + result = Curl_setstropt(&data->set.str[STRING_SSL_CIPHER13_LIST_ORIG], + va_arg(param, char *)); + } + else + return CURLE_NOT_BUILT_IN; + break; + case CURLOPT_PROXY_TLS13_CIPHERS: + if(Curl_ssl_tls13_ciphersuites()) { + /* set preferred list of TLS 1.3 cipher suites for proxy */ + result = Curl_setstropt(&data->set.str[STRING_SSL_CIPHER13_LIST_PROXY], + va_arg(param, char *)); + } + else + return CURLE_NOT_BUILT_IN; + break; + case CURLOPT_RANDOM_FILE: /* * This is the path name to a file that contains random data to seed @@ -1584,14 +1603,19 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, * String that holds the SSL crypto engine. */ argptr = va_arg(param, char *); - if(argptr && argptr[0]) - result = Curl_ssl_set_engine(data, argptr); + if(argptr && argptr[0]) { + result = Curl_setstropt(&data->set.str[STRING_SSL_ENGINE], argptr); + if(!result) { + result = Curl_ssl_set_engine(data, argptr); + } + } break; case CURLOPT_SSLENGINE_DEFAULT: /* * flag to set engine as default. */ + Curl_setstropt(&data->set.str[STRING_SSL_ENGINE], NULL); result = Curl_ssl_set_engine_default(data); break; case CURLOPT_CRLF: @@ -1942,6 +1966,11 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, if(data->share->sslsession == data->state.session) data->state.session = NULL; +#ifdef USE_LIBPSL + if(data->psl == &data->share->psl) + data->psl = data->multi? &data->multi->psl: NULL; +#endif + data->share->dirty--; Curl_share_unlock(data, CURL_LOCK_DATA_SHARE); @@ -1973,8 +2002,12 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, data->set.general_ssl.max_ssl_sessions = data->share->max_ssl_sessions; data->state.session = data->share->sslsession; } - Curl_share_unlock(data, CURL_LOCK_DATA_SHARE); +#ifdef USE_LIBPSL + if(data->share->specifier & (1 << CURL_LOCK_DATA_PSL)) + data->psl = &data->share->psl; +#endif + Curl_share_unlock(data, CURL_LOCK_DATA_SHARE); } /* check for host cache not needed, * it will be done by curl_easy_perform */ @@ -2562,6 +2595,10 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, case CURLOPT_DNS_SHUFFLE_ADDRESSES: data->set.dns_shuffle_addresses = (0 != va_arg(param, long)) ? TRUE:FALSE; break; + case CURLOPT_DISALLOW_USERNAME_IN_URL: + data->set.disallow_username_in_url = + (0 != va_arg(param, long)) ? TRUE : FALSE; + break; default: /* unknown tag and its companion, just ignore: */ result = CURLE_UNKNOWN_OPTION; diff --git a/Utilities/cmcurl/lib/sha256.c b/Utilities/cmcurl/lib/sha256.c index 55716c6..f9287af 100644 --- a/Utilities/cmcurl/lib/sha256.c +++ b/Utilities/cmcurl/lib/sha256.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Florin Petriuc, <petriuc.florin@gmail.com> + * Copyright (C) 1998 - 2018, Florin Petriuc, <petriuc.florin@gmail.com> * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -123,15 +123,11 @@ static const unsigned long K[64] = { #define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) #define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) #define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) -#ifndef MIN -#define MIN(x, y) (((x) < (y)) ? (x) : (y)) -#endif /* compress 512-bits */ static int sha256_compress(struct sha256_state *md, unsigned char *buf) { - unsigned long S[8], W[64], t0, t1; - unsigned long t; + unsigned long S[8], W[64]; int i; /* copy state into S */ for(i = 0; i < 8; i++) { @@ -146,12 +142,13 @@ static int sha256_compress(struct sha256_state *md, W[i - 16]; } /* Compress */ -#define RND(a,b,c,d,e,f,g,h,i) \ - t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ - t1 = Sigma0(a) + Maj(a, b, c); \ - d += t0; \ +#define RND(a,b,c,d,e,f,g,h,i) \ + unsigned long t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + unsigned long t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ h = t0 + t1; for(i = 0; i < 64; ++i) { + unsigned long t; RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], i); t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4]; S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t; @@ -200,7 +197,7 @@ static int SHA256_Update(struct sha256_state *md, inlen -= block_size; } else { - n = MIN(inlen, (block_size - md->curlen)); + n = CURLMIN(inlen, (block_size - md->curlen)); memcpy(md->buf + md->curlen, in, n); md->curlen += n; in += n; diff --git a/Utilities/cmcurl/lib/share.c b/Utilities/cmcurl/lib/share.c index 870b191..3d51086 100644 --- a/Utilities/cmcurl/lib/share.c +++ b/Utilities/cmcurl/lib/share.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -25,6 +25,7 @@ #include <curl/curl.h> #include "urldata.h" #include "share.h" +#include "psl.h" #include "vtls/vtls.h" #include "curl_memory.h" @@ -106,6 +107,12 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) res = CURLSHE_NOMEM; break; + case CURL_LOCK_DATA_PSL: +#ifndef USE_LIBPSL + res = CURLSHE_NOT_BUILT_IN; +#endif + break; + default: res = CURLSHE_BAD_OPTION; } @@ -205,6 +212,8 @@ curl_share_cleanup(struct Curl_share *share) } #endif + Curl_psl_destroy(&share->psl); + if(share->unlockfunc) share->unlockfunc(NULL, CURL_LOCK_DATA_SHARE, share->clientdata); free(share); diff --git a/Utilities/cmcurl/lib/share.h b/Utilities/cmcurl/lib/share.h index 4b13406..a7dea41 100644 --- a/Utilities/cmcurl/lib/share.h +++ b/Utilities/cmcurl/lib/share.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -25,6 +25,7 @@ #include "curl_setup.h" #include <curl/curl.h> #include "cookie.h" +#include "psl.h" #include "urldata.h" #include "conncache.h" @@ -49,6 +50,9 @@ struct Curl_share { #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES) struct CookieInfo *cookies; #endif +#ifdef USE_LIBPSL + struct PslCache psl; +#endif struct curl_ssl_session *sslsession; size_t max_ssl_sessions; diff --git a/Utilities/cmcurl/lib/smb.c b/Utilities/cmcurl/lib/smb.c index 9ac6150..e4b18fc 100644 --- a/Utilities/cmcurl/lib/smb.c +++ b/Utilities/cmcurl/lib/smb.c @@ -59,6 +59,7 @@ static CURLcode smb_setup_connection(struct connectdata *conn); static CURLcode smb_connect(struct connectdata *conn, bool *done); static CURLcode smb_connection_state(struct connectdata *conn, bool *done); +static CURLcode smb_do(struct connectdata *conn, bool *done); static CURLcode smb_request_state(struct connectdata *conn, bool *done); static CURLcode smb_done(struct connectdata *conn, CURLcode status, bool premature); @@ -73,7 +74,7 @@ static CURLcode smb_parse_url_path(struct connectdata *conn); const struct Curl_handler Curl_handler_smb = { "SMB", /* scheme */ smb_setup_connection, /* setup_connection */ - ZERO_NULL, /* do_it */ + smb_do, /* do_it */ smb_done, /* done */ ZERO_NULL, /* do_more */ smb_connect, /* connect_it */ @@ -98,7 +99,7 @@ const struct Curl_handler Curl_handler_smb = { const struct Curl_handler Curl_handler_smbs = { "SMBS", /* scheme */ smb_setup_connection, /* setup_connection */ - ZERO_NULL, /* do_it */ + smb_do, /* do_it */ smb_done, /* done */ ZERO_NULL, /* do_more */ smb_connect, /* connect_it */ @@ -173,7 +174,6 @@ enum smb_req_state { /* SMB request data */ struct smb_request { enum smb_req_state state; - char *share; char *path; unsigned short tid; /* Even if we connect to the same tree as another */ unsigned short fid; /* request, the tid will be different */ @@ -182,7 +182,7 @@ struct smb_request { static void conn_state(struct connectdata *conn, enum smb_conn_state newstate) { - struct smb_conn *smb = &conn->proto.smbc; + struct smb_conn *smbc = &conn->proto.smbc; #if defined(DEBUGBUILD) && !defined(CURL_DISABLE_VERBOSE_STRINGS) /* For debug purposes */ static const char * const names[] = { @@ -194,12 +194,12 @@ static void conn_state(struct connectdata *conn, enum smb_conn_state newstate) /* LAST */ }; - if(smb->state != newstate) + if(smbc->state != newstate) infof(conn->data, "SMB conn %p state change from %s to %s\n", - (void *)smb, names[smb->state], names[newstate]); + (void *)smbc, names[smbc->state], names[newstate]); #endif - smb->state = newstate; + smbc->state = newstate; } static void request_state(struct connectdata *conn, @@ -228,6 +228,8 @@ static void request_state(struct connectdata *conn, req->state = newstate; } +/* this should setup things in the connection, not in the easy + handle */ static CURLcode smb_setup_connection(struct connectdata *conn) { struct smb_request *req; @@ -253,7 +255,6 @@ static CURLcode smb_connect(struct connectdata *conn, bool *done) return CURLE_LOGIN_DENIED; /* Initialize the connection state */ - memset(smbc, 0, sizeof(*smbc)); smbc->state = SMB_CONNECTING; smbc->recv_buf = malloc(MAX_MESSAGE_SIZE); if(!smbc->recv_buf) @@ -366,7 +367,7 @@ static CURLcode smb_send(struct connectdata *conn, ssize_t len, ssize_t bytes_written; CURLcode result; - result = Curl_write(conn, FIRSTSOCKET, conn->data->state.uploadbuffer, + result = Curl_write(conn, FIRSTSOCKET, conn->data->state.ulbuf, len, &bytes_written); if(result) return result; @@ -392,7 +393,7 @@ static CURLcode smb_flush(struct connectdata *conn) return CURLE_OK; result = Curl_write(conn, FIRSTSOCKET, - conn->data->state.uploadbuffer + smbc->sent, + conn->data->state.ulbuf + smbc->sent, len, &bytes_written); if(result) return result; @@ -408,9 +409,12 @@ static CURLcode smb_flush(struct connectdata *conn) static CURLcode smb_send_message(struct connectdata *conn, unsigned char cmd, const void *msg, size_t msg_len) { - smb_format_message(conn, (struct smb_header *)conn->data->state.uploadbuffer, + CURLcode result = Curl_get_upload_buffer(conn->data); + if(result) + return result; + smb_format_message(conn, (struct smb_header *)conn->data->state.ulbuf, cmd, msg_len); - memcpy(conn->data->state.uploadbuffer + sizeof(struct smb_header), + memcpy(conn->data->state.ulbuf + sizeof(struct smb_header), msg, msg_len); return smb_send(conn, sizeof(struct smb_header) + msg_len, 0); @@ -475,11 +479,11 @@ static CURLcode smb_send_setup(struct connectdata *conn) static CURLcode smb_send_tree_connect(struct connectdata *conn) { - struct smb_request *req = conn->data->req.protop; struct smb_tree_connect msg; + struct smb_conn *smbc = &conn->proto.smbc; char *p = msg.bytes; - size_t byte_count = strlen(conn->host.name) + strlen(req->share); + size_t byte_count = strlen(conn->host.name) + strlen(smbc->share); byte_count += strlen(SERVICENAME) + 5; /* 2 nulls and 3 backslashes */ if(byte_count > sizeof(msg.bytes)) return CURLE_FILESIZE_EXCEEDED; @@ -491,7 +495,7 @@ static CURLcode smb_send_tree_connect(struct connectdata *conn) MSGCAT("\\\\"); MSGCAT(conn->host.name); MSGCAT("\\"); - MSGCATNULL(req->share); + MSGCATNULL(smbc->share); MSGCATNULL(SERVICENAME); /* Match any type of service */ byte_count = p - msg.bytes; msg.byte_count = smb_swap16((unsigned short)byte_count); @@ -571,11 +575,15 @@ static CURLcode smb_send_read(struct connectdata *conn) static CURLcode smb_send_write(struct connectdata *conn) { - struct smb_write *msg = (struct smb_write *)conn->data->state.uploadbuffer; + struct smb_write *msg; struct smb_request *req = conn->data->req.protop; curl_off_t offset = conn->data->req.offset; - curl_off_t upload_size = conn->data->req.size - conn->data->req.bytecount; + CURLcode result = Curl_get_upload_buffer(conn->data); + if(result) + return result; + msg = (struct smb_write *)conn->data->state.ulbuf; + if(upload_size >= MAX_PAYLOAD_SIZE - 1) /* There is one byte of padding */ upload_size = MAX_PAYLOAD_SIZE - 1; @@ -602,9 +610,9 @@ static CURLcode smb_send_and_recv(struct connectdata *conn, void **msg) /* Check if there is data in the transfer buffer */ if(!smbc->send_size && smbc->upload_size) { - int nread = smbc->upload_size > UPLOAD_BUFSIZE ? UPLOAD_BUFSIZE : - (int) smbc->upload_size; - conn->data->req.upload_fromhere = conn->data->state.uploadbuffer; + size_t nread = smbc->upload_size > UPLOAD_BUFSIZE ? UPLOAD_BUFSIZE : + smbc->upload_size; + conn->data->req.upload_fromhere = conn->data->state.ulbuf; result = Curl_fillreadbuffer(conn, nread, &nread); if(result && result != CURLE_AGAIN) return result; @@ -910,55 +918,52 @@ static CURLcode smb_request_state(struct connectdata *conn, bool *done) static CURLcode smb_done(struct connectdata *conn, CURLcode status, bool premature) { - struct smb_request *req = conn->data->req.protop; - (void) premature; - - Curl_safefree(req->share); Curl_safefree(conn->data->req.protop); - return status; } static CURLcode smb_disconnect(struct connectdata *conn, bool dead) { struct smb_conn *smbc = &conn->proto.smbc; - struct smb_request *req = conn->data->req.protop; - (void) dead; - + Curl_safefree(smbc->share); Curl_safefree(smbc->domain); Curl_safefree(smbc->recv_buf); - - /* smb_done is not always called, so cleanup the request */ - if(req) { - Curl_safefree(req->share); - } - return CURLE_OK; } static int smb_getsock(struct connectdata *conn, curl_socket_t *socks, int numsocks) { - struct smb_conn *smbc = &conn->proto.smbc; - if(!numsocks) return GETSOCK_BLANK; socks[0] = conn->sock[FIRSTSOCKET]; + return GETSOCK_READSOCK(0) | GETSOCK_WRITESOCK(0); +} - if(smbc->send_size || smbc->upload_size) - return GETSOCK_WRITESOCK(0); +static CURLcode smb_do(struct connectdata *conn, bool *done) +{ + struct smb_conn *smbc = &conn->proto.smbc; + struct smb_request *req = conn->data->req.protop; - return GETSOCK_READSOCK(0); + *done = FALSE; + if(smbc->share) { + req->path = strchr(smbc->share, '\0'); + if(req->path) { + req->path++; + return CURLE_OK; + } + } + return CURLE_URL_MALFORMAT; } static CURLcode smb_parse_url_path(struct connectdata *conn) { CURLcode result = CURLE_OK; struct Curl_easy *data = conn->data; - struct smb_request *req = data->req.protop; + struct smb_conn *smbc = &conn->proto.smbc; char *path; char *slash; @@ -968,35 +973,29 @@ static CURLcode smb_parse_url_path(struct connectdata *conn) return result; /* Parse the path for the share */ - req->share = strdup((*path == '/' || *path == '\\') ? path + 1 : path); - if(!req->share) { - free(path); - + smbc->share = strdup((*path == '/' || *path == '\\') ? path + 1 : path); + free(path); + if(!smbc->share) return CURLE_OUT_OF_MEMORY; - } - slash = strchr(req->share, '/'); + slash = strchr(smbc->share, '/'); if(!slash) - slash = strchr(req->share, '\\'); + slash = strchr(smbc->share, '\\'); /* The share must be present */ if(!slash) { - free(path); - + Curl_safefree(smbc->share); return CURLE_URL_MALFORMAT; } /* Parse the path for the file path converting any forward slashes into backslashes */ *slash++ = 0; - req->path = slash; + for(; *slash; slash++) { if(*slash == '/') *slash = '\\'; } - - free(path); - return CURLE_OK; } diff --git a/Utilities/cmcurl/lib/smb.h b/Utilities/cmcurl/lib/smb.h index c3ee7ae..9ce6b56 100644 --- a/Utilities/cmcurl/lib/smb.h +++ b/Utilities/cmcurl/lib/smb.h @@ -35,6 +35,7 @@ struct smb_conn { enum smb_conn_state state; char *user; char *domain; + char *share; unsigned char challenge[8]; unsigned int session_key; unsigned short uid; diff --git a/Utilities/cmcurl/lib/smtp.c b/Utilities/cmcurl/lib/smtp.c index 3f3b45a..ecf10a4 100644 --- a/Utilities/cmcurl/lib/smtp.c +++ b/Utilities/cmcurl/lib/smtp.c @@ -704,7 +704,6 @@ static CURLcode smtp_state_ehlo_resp(struct connectdata *conn, int smtpcode, struct smtp_conn *smtpc = &conn->proto.smtpc; const char *line = data->state.buffer; size_t len = strlen(line); - size_t wordlen; (void)instate; /* no use for this yet */ @@ -739,6 +738,7 @@ static CURLcode smtp_state_ehlo_resp(struct connectdata *conn, int smtpcode, /* Loop through the data line */ for(;;) { size_t llen; + size_t wordlen; unsigned int mechbit; while(len && @@ -1563,13 +1563,14 @@ CURLcode Curl_smtp_escape_eob(struct connectdata *conn, const ssize_t nread) if(!scratch || data->set.crlf) { oldscratch = scratch; - scratch = newscratch = malloc(2 * data->set.buffer_size); + scratch = newscratch = malloc(2 * UPLOAD_BUFSIZE); if(!newscratch) { failf(data, "Failed to alloc scratch buffer!"); return CURLE_OUT_OF_MEMORY; } } + DEBUGASSERT(UPLOAD_BUFSIZE >= nread); /* Have we already sent part of the EOB? */ eob_sent = smtp->eob; diff --git a/Utilities/cmcurl/lib/socks.c b/Utilities/cmcurl/lib/socks.c index ac4270e..81f3eda 100644 --- a/Utilities/cmcurl/lib/socks.c +++ b/Utilities/cmcurl/lib/socks.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -57,10 +57,9 @@ int Curl_blockread_all(struct connectdata *conn, /* connection data */ ssize_t nread; ssize_t allread = 0; int result; - timediff_t timeleft; *n = 0; for(;;) { - timeleft = Curl_timeleft(conn->data, NULL, TRUE); + timediff_t timeleft = Curl_timeleft(conn->data, NULL, TRUE); if(timeleft < 0) { /* we already got the timeout */ result = CURLE_OPERATION_TIMEDOUT; @@ -116,7 +115,6 @@ CURLcode Curl_SOCKS4(const char *proxy_user, #define SOCKS4REQLEN 262 unsigned char socksreq[SOCKS4REQLEN]; /* room for SOCKS4 request incl. user id */ - int result; CURLcode code; curl_socket_t sock = conn->sock[sockindex]; struct Curl_easy *data = conn->data; @@ -220,11 +218,12 @@ CURLcode Curl_SOCKS4(const char *proxy_user, * Make connection */ { + int result; ssize_t actualread; ssize_t written; ssize_t hostnamelen = 0; - int packetsize = 9 + - (int)strlen((char *)socksreq + 8); /* size including NUL */ + ssize_t packetsize = 9 + + strlen((char *)socksreq + 8); /* size including NUL */ /* If SOCKS4a, set special invalid IP address 0.0.0.x */ if(protocol4a) { @@ -617,11 +616,11 @@ CURLcode Curl_SOCKS5(const char *proxy_user, if(dns) hp = dns->addr; if(hp) { - int i; char buf[64]; Curl_printable_address(hp, buf, sizeof(buf)); if(hp->ai_family == AF_INET) { + int i; struct sockaddr_in *saddr_in; socksreq[len++] = 1; /* ATYP: IPv4 = 1 */ @@ -634,6 +633,7 @@ CURLcode Curl_SOCKS5(const char *proxy_user, } #ifdef ENABLE_IPV6 else if(hp->ai_family == AF_INET6) { + int i; struct sockaddr_in6 *saddr_in6; socksreq[len++] = 4; /* ATYP: IPv6 = 4 */ diff --git a/Utilities/cmcurl/lib/splay.c b/Utilities/cmcurl/lib/splay.c index 69af446..c54a63b 100644 --- a/Utilities/cmcurl/lib/splay.c +++ b/Utilities/cmcurl/lib/splay.c @@ -41,7 +41,6 @@ struct Curl_tree *Curl_splay(struct curltime i, struct Curl_tree *t) { struct Curl_tree N, *l, *r, *y; - long comp; if(t == NULL) return t; @@ -49,7 +48,7 @@ struct Curl_tree *Curl_splay(struct curltime i, l = r = &N; for(;;) { - comp = compare(i, t->key); + long comp = compare(i, t->key); if(comp < 0) { if(t->smaller == NULL) break; diff --git a/Utilities/cmcurl/lib/ssh-libssh.c b/Utilities/cmcurl/lib/ssh-libssh.c index 34ef490..7d59089 100644 --- a/Utilities/cmcurl/lib/ssh-libssh.c +++ b/Utilities/cmcurl/lib/ssh-libssh.c @@ -204,11 +204,21 @@ static CURLcode sftp_error_to_CURLE(int err) return CURLE_SSH; } +#ifndef DEBUGBUILD +#define state(x,y) mystate(x,y) +#else +#define state(x,y) mystate(x,y, __LINE__) +#endif + /* * SSH State machine related code */ /* This is the ONLY way to change SSH state! */ -static void state(struct connectdata *conn, sshstate nowstate) +static void mystate(struct connectdata *conn, sshstate nowstate +#ifdef DEBUGBUILD + , int lineno +#endif + ) { struct ssh_conn *sshc = &conn->proto.sshc; #if defined(DEBUGBUILD) && !defined(CURL_DISABLE_VERBOSE_STRINGS) @@ -278,8 +288,9 @@ static void state(struct connectdata *conn, sshstate nowstate) if(sshc->state != nowstate) { - infof(conn->data, "SSH %p state change from %s to %s\n", - (void *) sshc, names[sshc->state], names[nowstate]); + infof(conn->data, "SSH %p state change from %s to %s (line %d)\n", + (void *) sshc, names[sshc->state], names[nowstate], + lineno); } #endif @@ -418,7 +429,7 @@ cleanup: } #define MOVE_TO_ERROR_STATE(_r) { \ - state(conn, SSH_SESSION_FREE); \ + state(conn, SSH_SESSION_DISCONNECT); \ sshc->actualcode = _r; \ rc = SSH_ERROR; \ break; \ @@ -486,7 +497,7 @@ restart: if(rc < 0) return SSH_ERROR; - /* fallthrough */ + /* FALLTHROUGH */ case 1: sshc->kbd_state = 1; @@ -561,7 +572,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) ssh_set_blocking(sshc->ssh_session, 0); state(conn, SSH_S_STARTUP); - /* fall-through */ + /* FALLTHROUGH */ case SSH_S_STARTUP: rc = ssh_connect(sshc->ssh_session); @@ -575,7 +586,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) state(conn, SSH_HOSTKEY); - /* fall-through */ + /* FALLTHROUGH */ case SSH_HOSTKEY: rc = myssh_is_known(conn); @@ -584,7 +595,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) } state(conn, SSH_AUTHLIST); - /* fall through */ + /* FALLTHROUGH */ case SSH_AUTHLIST:{ sshc->authed = FALSE; @@ -607,6 +618,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) sshc->auth_methods = ssh_userauth_list(sshc->ssh_session, NULL); if(sshc->auth_methods & SSH_AUTH_METHOD_PUBLICKEY) { state(conn, SSH_AUTH_PKEY_INIT); + infof(data, "Authentication using SSH public key file\n"); } else if(sshc->auth_methods & SSH_AUTH_METHOD_GSSAPI_MIC) { state(conn, SSH_AUTH_GSSAPI); @@ -651,6 +663,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) if(rc != SSH_OK) { failf(data, "Could not load private key file %s", data->set.str[STRING_SSH_PRIVATE_KEY]); + MOVE_TO_ERROR_STATE(CURLE_LOGIN_DENIED); break; } @@ -659,8 +672,6 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) } else { - infof(data, "Authentication using SSH public key file\n"); - rc = ssh_userauth_publickey_auto(sshc->ssh_session, NULL, data->set.ssl.key_passwd); if(rc == SSH_AUTH_AGAIN) { @@ -748,7 +759,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) MOVE_TO_ERROR_STATE(CURLE_LOGIN_DENIED); } state(conn, SSH_AUTH_PASS); - /* fall through */ + /* FALLTHROUGH */ case SSH_AUTH_PASS: rc = ssh_userauth_password(sshc->ssh_session, NULL, conn->passwd); @@ -812,7 +823,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) break; } state(conn, SSH_SFTP_REALPATH); - /* fall through */ + /* FALLTHROUGH */ case SSH_SFTP_REALPATH: /* * Get the "home" directory @@ -1279,7 +1290,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) if(sshc->readdir_attrs) { sshc->readdir_filename = sshc->readdir_attrs->name; sshc->readdir_longentry = sshc->readdir_attrs->longname; - sshc->readdir_len = (int)strlen(sshc->readdir_filename); + sshc->readdir_len = strlen(sshc->readdir_filename); if(data->set.ftp_list_only) { char *tmpLine; @@ -1306,11 +1317,11 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) if(data->set.verbose) { Curl_debug(data, CURLINFO_DATA_OUT, (char *)sshc->readdir_filename, - sshc->readdir_len, conn); + sshc->readdir_len); } } else { - sshc->readdir_currLen = (int)strlen(sshc->readdir_longentry); + sshc->readdir_currLen = strlen(sshc->readdir_longentry); sshc->readdir_totalLen = 80 + sshc->readdir_currLen; sshc->readdir_line = calloc(sshc->readdir_totalLen, 1); if(!sshc->readdir_line) { @@ -1371,12 +1382,12 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) if(sshc->readdir_filename == NULL) sshc->readdir_len = 0; else - sshc->readdir_len = (int)strlen(sshc->readdir_tmp); + sshc->readdir_len = strlen(sshc->readdir_tmp); sshc->readdir_longentry = NULL; sshc->readdir_filename = sshc->readdir_tmp; } else { - sshc->readdir_len = (int)strlen(sshc->readdir_link_attrs->name); + sshc->readdir_len = strlen(sshc->readdir_link_attrs->name); sshc->readdir_filename = sshc->readdir_link_attrs->name; sshc->readdir_longentry = sshc->readdir_link_attrs->longname; } @@ -1408,7 +1419,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) sshc->readdir_longentry = NULL; state(conn, SSH_SFTP_READDIR_BOTTOM); - /* fall through */ + /* FALLTHROUGH */ case SSH_SFTP_READDIR_BOTTOM: sshc->readdir_currLen += snprintf(sshc->readdir_line + sshc->readdir_currLen, @@ -1423,7 +1434,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) /* output debug output if that is requested */ if(data->set.verbose) { Curl_debug(data, CURLINFO_DATA_OUT, sshc->readdir_line, - sshc->readdir_currLen, conn); + sshc->readdir_currLen); } data->req.bytecount += sshc->readdir_currLen; } @@ -1740,7 +1751,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) MOVE_TO_ERROR_STATE(CURLE_COULDNT_CONNECT); } state(conn, SSH_SCP_DOWNLOAD); - /* fall through */ + /* FALLTHROUGH */ case SSH_SCP_DOWNLOAD:{ curl_off_t bytecount; @@ -1805,7 +1816,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) ssh_set_blocking(sshc->ssh_session, 0); state(conn, SSH_SESSION_DISCONNECT); - /* fall through */ + /* FALLTHROUGH */ case SSH_SESSION_DISCONNECT: /* during weird times when we've been prematurely aborted, the channel @@ -1822,7 +1833,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) conn->data->state.most_recent_ftp_entrypath = NULL; state(conn, SSH_SESSION_FREE); - /* fall through */ + /* FALLTHROUGH */ case SSH_SESSION_FREE: if(sshc->ssh_session) { ssh_free(sshc->ssh_session); @@ -2379,7 +2390,8 @@ static CURLcode sftp_done(struct connectdata *conn, CURLcode status, /* Post quote commands are executed after the SFTP_CLOSE state to avoid errors that could happen due to open file handles during POSTQUOTE operation */ - if(!status && !premature && conn->data->set.postquote) { + if(!status && !premature && conn->data->set.postquote && + !conn->bits.retry) { sshc->nextstate = SSH_SFTP_POSTQUOTE_INIT; state(conn, SSH_SFTP_CLOSE); } @@ -2437,7 +2449,7 @@ static ssize_t sftp_recv(struct connectdata *conn, int sockindex, return -1; } - /* fall-through */ + /* FALLTHROUGH */ case 1: conn->proto.sshc.sftp_recv_state = 1; @@ -2500,8 +2512,8 @@ static void sftp_quote(struct connectdata *conn) return; } if(data->set.verbose) { - Curl_debug(data, CURLINFO_HEADER_OUT, (char *) "PWD\n", 4, conn); - Curl_debug(data, CURLINFO_HEADER_IN, tmp, strlen(tmp), conn); + Curl_debug(data, CURLINFO_HEADER_OUT, (char *) "PWD\n", 4); + Curl_debug(data, CURLINFO_HEADER_IN, tmp, strlen(tmp)); } /* this sends an FTP-like "header" to the header callback so that the current directory can be read very similar to how it is read when diff --git a/Utilities/cmcurl/lib/ssh.c b/Utilities/cmcurl/lib/ssh.c index bf7bd54..a4b2ca4 100644 --- a/Utilities/cmcurl/lib/ssh.c +++ b/Utilities/cmcurl/lib/ssh.c @@ -659,7 +659,7 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) libssh2_session_set_blocking(sshc->ssh_session, 0); state(conn, SSH_S_STARTUP); - /* fall-through */ + /* FALLTHROUGH */ case SSH_S_STARTUP: rc = libssh2_session_startup(sshc->ssh_session, (int)sock); @@ -675,7 +675,7 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) state(conn, SSH_HOSTKEY); - /* fall-through */ + /* FALLTHROUGH */ case SSH_HOSTKEY: /* * Before we authenticate we should check the hostkey's fingerprint @@ -1175,8 +1175,8 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) break; } if(data->set.verbose) { - Curl_debug(data, CURLINFO_HEADER_OUT, (char *)"PWD\n", 4, conn); - Curl_debug(data, CURLINFO_HEADER_IN, tmp, strlen(tmp), conn); + Curl_debug(data, CURLINFO_HEADER_OUT, (char *)"PWD\n", 4); + Curl_debug(data, CURLINFO_HEADER_IN, tmp, strlen(tmp)); } /* this sends an FTP-like "header" to the header callback so that the current directory can be read very similar to how it is read when @@ -1933,17 +1933,17 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) break; case SSH_SFTP_READDIR: - sshc->readdir_len = libssh2_sftp_readdir_ex(sshc->sftp_handle, - sshc->readdir_filename, - PATH_MAX, - sshc->readdir_longentry, - PATH_MAX, - &sshc->readdir_attrs); - if(sshc->readdir_len == LIBSSH2_ERROR_EAGAIN) { - rc = LIBSSH2_ERROR_EAGAIN; + rc = libssh2_sftp_readdir_ex(sshc->sftp_handle, + sshc->readdir_filename, + PATH_MAX, + sshc->readdir_longentry, + PATH_MAX, + &sshc->readdir_attrs); + if(rc == LIBSSH2_ERROR_EAGAIN) { break; } - if(sshc->readdir_len > 0) { + if(rc > 0) { + sshc->readdir_len = (size_t) rc; sshc->readdir_filename[sshc->readdir_len] = '\0'; if(data->set.ftp_list_only) { @@ -1970,11 +1970,11 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) /* output debug output if that is requested */ if(data->set.verbose) { Curl_debug(data, CURLINFO_DATA_OUT, sshc->readdir_filename, - sshc->readdir_len, conn); + sshc->readdir_len); } } else { - sshc->readdir_currLen = (int)strlen(sshc->readdir_longentry); + sshc->readdir_currLen = strlen(sshc->readdir_longentry); sshc->readdir_totalLen = 80 + sshc->readdir_currLen; sshc->readdir_line = calloc(sshc->readdir_totalLen, 1); if(!sshc->readdir_line) { @@ -2008,13 +2008,13 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) break; } } - else if(sshc->readdir_len == 0) { + else if(rc == 0) { Curl_safefree(sshc->readdir_filename); Curl_safefree(sshc->readdir_longentry); state(conn, SSH_SFTP_READDIR_DONE); break; } - else if(sshc->readdir_len <= 0) { + else if(rc < 0) { err = sftp_libssh2_last_error(sshc->sftp_session); result = sftp_libssh2_error_to_CURLE(err); sshc->actualcode = result?result:CURLE_SSH; @@ -2029,16 +2029,16 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) break; case SSH_SFTP_READDIR_LINK: - sshc->readdir_len = + rc = libssh2_sftp_symlink_ex(sshc->sftp_session, sshc->readdir_linkPath, curlx_uztoui(strlen(sshc->readdir_linkPath)), sshc->readdir_filename, PATH_MAX, LIBSSH2_SFTP_READLINK); - if(sshc->readdir_len == LIBSSH2_ERROR_EAGAIN) { - rc = LIBSSH2_ERROR_EAGAIN; + if(rc == LIBSSH2_ERROR_EAGAIN) { break; } + sshc->readdir_len = (size_t) rc; Curl_safefree(sshc->readdir_linkPath); /* get room for the filename and extra output */ @@ -2079,7 +2079,7 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) /* output debug output if that is requested */ if(data->set.verbose) { Curl_debug(data, CURLINFO_DATA_OUT, sshc->readdir_line, - sshc->readdir_currLen, conn); + sshc->readdir_currLen); } data->req.bytecount += sshc->readdir_currLen; } @@ -3219,7 +3219,8 @@ static CURLcode sftp_done(struct connectdata *conn, CURLcode status, /* Post quote commands are executed after the SFTP_CLOSE state to avoid errors that could happen due to open file handles during POSTQUOTE operation */ - if(!status && !premature && conn->data->set.postquote) { + if(!status && !premature && conn->data->set.postquote && + !conn->bits.retry) { sshc->nextstate = SSH_SFTP_POSTQUOTE_INIT; state(conn, SSH_SFTP_CLOSE); } diff --git a/Utilities/cmcurl/lib/ssh.h b/Utilities/cmcurl/lib/ssh.h index 1c13550..0620aac 100644 --- a/Utilities/cmcurl/lib/ssh.h +++ b/Utilities/cmcurl/lib/ssh.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -131,7 +131,7 @@ struct ssh_conn { quote command fails) */ char *homedir; /* when doing SFTP we figure out home dir in the connect phase */ - int readdir_len, readdir_totalLen, readdir_currLen; + size_t readdir_len, readdir_totalLen, readdir_currLen; char *readdir_line; char *readdir_linkPath; /* end of READDIR stuff */ diff --git a/Utilities/cmcurl/lib/strcase.h b/Utilities/cmcurl/lib/strcase.h index ea2abc8..6fee384 100644 --- a/Utilities/cmcurl/lib/strcase.h +++ b/Utilities/cmcurl/lib/strcase.h @@ -46,6 +46,5 @@ char Curl_raw_toupper(char in); #define checkprefix(a,b) curl_strnequal(a,b,strlen(a)) void Curl_strntoupper(char *dest, const char *src, size_t n); -char Curl_raw_toupper(char in); #endif /* HEADER_CURL_STRCASE_H */ diff --git a/Utilities/cmcurl/lib/system_win32.c b/Utilities/cmcurl/lib/system_win32.c index cfbbf32..6b8004e 100644 --- a/Utilities/cmcurl/lib/system_win32.c +++ b/Utilities/cmcurl/lib/system_win32.c @@ -26,6 +26,7 @@ #include <curl/curl.h> #include "system_win32.h" +#include "warnless.h" /* The last #include files should be: */ #include "curl_memory.h" @@ -134,8 +135,9 @@ bool Curl_verify_windows_version(const unsigned int majorVersion, break; case VERSION_LESS_THAN_EQUAL: - if(osver.dwMajorVersion <= majorVersion && - osver.dwMinorVersion <= minorVersion) + if(osver.dwMajorVersion < majorVersion || + (osver.dwMajorVersion == majorVersion && + osver.dwMinorVersion <= minorVersion)) matched = TRUE; break; @@ -146,8 +148,9 @@ bool Curl_verify_windows_version(const unsigned int majorVersion, break; case VERSION_GREATER_THAN_EQUAL: - if(osver.dwMajorVersion >= majorVersion && - osver.dwMinorVersion >= minorVersion) + if(osver.dwMajorVersion > majorVersion || + (osver.dwMajorVersion == majorVersion && + osver.dwMinorVersion >= minorVersion)) matched = TRUE; break; @@ -278,7 +281,9 @@ HMODULE Curl_load_library(LPCTSTR filename) /* Attempt to find LoadLibraryEx() which is only available on Windows 2000 and above */ - pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBARYEX); + pLoadLibraryEx = + CURLX_FUNCTION_CAST(LOADLIBRARYEX_FN, + (GetProcAddress(hKernel32, LOADLIBARYEX))); /* Detect if there's already a path in the filename and load the library if there is. Note: Both back slashes and forward slashes have been supported diff --git a/Utilities/cmcurl/lib/telnet.c b/Utilities/cmcurl/lib/telnet.c index d71c8e0..05fe744 100644 --- a/Utilities/cmcurl/lib/telnet.c +++ b/Utilities/cmcurl/lib/telnet.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -52,10 +52,6 @@ #include "connect.h" #include "progress.h" #include "system_win32.h" - -#define TELOPTS -#define TELCMDS - #include "arpa_telnet.h" #include "select.h" #include "strcase.h" @@ -74,10 +70,10 @@ x->subend = x->subpointer; \ CURL_SB_CLEAR(x); \ } WHILE_FALSE -#define CURL_SB_ACCUM(x,c) \ - do { \ - if(x->subpointer < (x->subbuffer + sizeof x->subbuffer)) \ - *x->subpointer++ = (c); \ +#define CURL_SB_ACCUM(x,c) \ + do { \ + if(x->subpointer < (x->subbuffer + sizeof(x->subbuffer))) \ + *x->subpointer++ = (c); \ } WHILE_FALSE #define CURL_SB_GET(x) ((*x->subpointer++)&0xff) @@ -92,6 +88,7 @@ #endif #ifdef USE_WINSOCK +typedef WSAEVENT (WINAPI *WSOCK2_EVENT)(void); typedef FARPROC WSOCK2_FUNC; static CURLcode check_wsock2(struct Curl_easy *data); #endif @@ -109,8 +106,10 @@ static void printoption(struct Curl_easy *data, static void negotiate(struct connectdata *); static void send_negotiation(struct connectdata *, int cmd, int option); -static void set_local_option(struct connectdata *, int cmd, int option); -static void set_remote_option(struct connectdata *, int cmd, int option); +static void set_local_option(struct connectdata *conn, + int option, int newstate); +static void set_remote_option(struct connectdata *conn, + int option, int newstate); static void printsub(struct Curl_easy *data, int direction, unsigned char *pointer, @@ -311,9 +310,6 @@ static void negotiate(struct connectdata *conn) static void printoption(struct Curl_easy *data, const char *direction, int cmd, int option) { - const char *fmt; - const char *opt; - if(data->set.verbose) { if(cmd == CURL_IAC) { if(CURL_TELCMD_OK(option)) @@ -322,9 +318,12 @@ static void printoption(struct Curl_easy *data, infof(data, "%s IAC %d\n", direction, option); } else { - fmt = (cmd == CURL_WILL) ? "WILL" : (cmd == CURL_WONT) ? "WONT" : - (cmd == CURL_DO) ? "DO" : (cmd == CURL_DONT) ? "DONT" : 0; + const char *fmt = (cmd == CURL_WILL) ? "WILL" : + (cmd == CURL_WONT) ? "WONT" : + (cmd == CURL_DO) ? "DO" : + (cmd == CURL_DONT) ? "DONT" : 0; if(fmt) { + const char *opt; if(CURL_TELOPT_OK(option)) opt = CURL_TELOPT(option); else if(option == CURL_TELOPT_EXOPL) @@ -348,7 +347,6 @@ static void send_negotiation(struct connectdata *conn, int cmd, int option) { unsigned char buf[3]; ssize_t bytes_written; - int err; struct Curl_easy *data = conn->data; buf[0] = CURL_IAC; @@ -357,7 +355,7 @@ static void send_negotiation(struct connectdata *conn, int cmd, int option) bytes_written = swrite(conn->sock[FIRSTSOCKET], buf, 3); if(bytes_written < 0) { - err = SOCKERRNO; + int err = SOCKERRNO; failf(data,"Sending data failed (%d)",err); } @@ -710,9 +708,8 @@ static void printsub(struct Curl_easy *data, unsigned char *pointer, /* where suboption data is */ size_t length) /* length of suboption data */ { - unsigned int i = 0; - if(data->set.verbose) { + unsigned int i = 0; if(direction) { infof(data, "%s IAC SB ", (direction == '<')? "RCVD":"SENT"); if(length >= 3) { @@ -764,7 +761,7 @@ static void printsub(struct Curl_easy *data, switch(pointer[0]) { case CURL_TELOPT_NAWS: if(length > 4) - infof(data, "Width: %hu ; Height: %hu", (pointer[1]<<8) | pointer[2], + infof(data, "Width: %d ; Height: %d", (pointer[1]<<8) | pointer[2], (pointer[3]<<8) | pointer[4]); break; default: @@ -928,7 +925,6 @@ static void suboption(struct connectdata *conn) unsigned char temp[2048]; ssize_t bytes_written; size_t len; - size_t tmplen; int err; char varname[128] = ""; char varval[128] = ""; @@ -968,7 +964,7 @@ static void suboption(struct connectdata *conn) len = 4; for(v = tn->telnet_vars; v; v = v->next) { - tmplen = (strlen(v->data) + 1); + size_t tmplen = (strlen(v->data) + 1); /* Add the variable only if it fits */ if(len + tmplen < (int)sizeof(temp)-6) { if(sscanf(v->data, "%127[^,],%127s", varname, varval)) { @@ -1017,7 +1013,7 @@ static void sendsuboption(struct connectdata *conn, int option) CURL_SB_ACCUM(tn, CURL_IAC); CURL_SB_ACCUM(tn, CURL_SB); CURL_SB_ACCUM(tn, CURL_TELOPT_NAWS); - /* We must deal either with litte or big endian processors */ + /* We must deal either with little or big endian processors */ /* Window size must be sent according to the 'network order' */ x = htons(tn->subopt_wsx); y = htons(tn->subopt_wsy); @@ -1223,7 +1219,7 @@ CURLcode telrcv(struct connectdata *conn, static CURLcode send_telnet_data(struct connectdata *conn, char *buffer, ssize_t nread) { - ssize_t escapes, i, j, outlen; + ssize_t escapes, i, outlen; unsigned char *outbuf = NULL; CURLcode result = CURLE_OK; ssize_t bytes_written, total_written; @@ -1238,6 +1234,7 @@ static CURLcode send_telnet_data(struct connectdata *conn, if(outlen == nread) outbuf = (unsigned char *)buffer; else { + ssize_t j; outbuf = malloc(nread + escapes + 1); if(!outbuf) return CURLE_OUT_OF_MEMORY; @@ -1306,7 +1303,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) #ifdef USE_WINSOCK HMODULE wsock2; WSOCK2_FUNC close_event_func; - WSOCK2_FUNC create_event_func; + WSOCK2_EVENT create_event_func; WSOCK2_FUNC event_select_func; WSOCK2_FUNC enum_netevents_func; WSAEVENT event_handle; @@ -1315,7 +1312,6 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) HANDLE objs[2]; DWORD obj_count; DWORD wait_timeout; - DWORD waitret; DWORD readfile_read; int err; #else @@ -1361,7 +1357,9 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) } /* Grab a pointer to WSACreateEvent */ - create_event_func = GetProcAddress(wsock2, "WSACreateEvent"); + create_event_func = + CURLX_FUNCTION_CAST(WSOCK2_EVENT, + (GetProcAddress(wsock2, "WSACreateEvent"))); if(create_event_func == NULL) { failf(data, "failed to find WSACreateEvent function (%u)", GetLastError()); FreeLibrary(wsock2); @@ -1438,7 +1436,8 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) /* Keep on listening and act on events */ while(keepon) { const DWORD buf_size = (DWORD)data->set.buffer_size; - waitret = WaitForMultipleObjects(obj_count, objs, FALSE, wait_timeout); + DWORD waitret = WaitForMultipleObjects(obj_count, objs, + FALSE, wait_timeout); switch(waitret) { case WAIT_TIMEOUT: { @@ -1607,7 +1606,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) case 0: /* timeout */ pfd[0].revents = 0; pfd[1].revents = 0; - /* fall through */ + /* FALLTHROUGH */ default: /* read! */ if(pfd[0].revents & POLLIN) { /* read data from network */ diff --git a/Utilities/cmcurl/lib/tftp.c b/Utilities/cmcurl/lib/tftp.c index b32960f..e5bc80b 100644 --- a/Utilities/cmcurl/lib/tftp.c +++ b/Utilities/cmcurl/lib/tftp.c @@ -451,7 +451,6 @@ static CURLcode tftp_send_first(tftp_state_data_t *state, tftp_event_t event) ssize_t senddata; const char *mode = "octet"; char *filename; - char buf[64]; struct Curl_easy *data = state->conn->data; CURLcode result = CURLE_OK; @@ -504,6 +503,7 @@ static CURLcode tftp_send_first(tftp_state_data_t *state, tftp_event_t event) /* optional addition of TFTP options */ if(!data->set.tftp_no_options) { + char buf[64]; /* add tsize option */ if(data->set.upload && (data->state.infilesize != -1)) snprintf(buf, sizeof(buf), "%" CURL_FORMAT_CURL_OFF_T, @@ -710,10 +710,9 @@ static CURLcode tftp_tx(tftp_state_data_t *state, tftp_event_t event) { struct Curl_easy *data = state->conn->data; ssize_t sbytes; - int rblock; CURLcode result = CURLE_OK; struct SingleRequest *k = &data->req; - int cb; /* Bytes currently read */ + size_t cb; /* Bytes currently read */ switch(event) { @@ -721,7 +720,7 @@ static CURLcode tftp_tx(tftp_state_data_t *state, tftp_event_t event) case TFTP_EVENT_OACK: if(event == TFTP_EVENT_ACK) { /* Ack the packet */ - rblock = getrpacketblock(&state->rpacket); + int rblock = getrpacketblock(&state->rpacket); if(rblock != state->block && /* There's a bug in tftpd-hpa that causes it to send us an ack for @@ -766,7 +765,7 @@ static CURLcode tftp_tx(tftp_state_data_t *state, tftp_event_t event) state->retries = 0; setpacketevent(&state->spacket, TFTP_EVENT_DATA); setpacketblock(&state->spacket, state->block); - if(state->block > 1 && state->sbytes < (int)state->blksize) { + if(state->block > 1 && state->sbytes < state->blksize) { state->state = TFTP_STATE_FIN; return CURLE_OK; } @@ -782,7 +781,7 @@ static CURLcode tftp_tx(tftp_state_data_t *state, tftp_event_t event) &cb); if(result) return result; - state->sbytes += cb; + state->sbytes += (int)cb; state->conn->data->req.upload_fromhere += cb; } while(state->sbytes < state->blksize && cb != 0); @@ -969,7 +968,7 @@ static CURLcode tftp_disconnect(struct connectdata *conn, bool dead_connection) static CURLcode tftp_connect(struct connectdata *conn, bool *done) { tftp_state_data_t *state; - int blksize, rc; + int blksize; blksize = TFTP_BLKSIZE_DEFAULT; @@ -1028,8 +1027,8 @@ static CURLcode tftp_connect(struct connectdata *conn, bool *done) * assume uses the same IP version and thus hopefully this works for both * IPv4 and IPv6... */ - rc = bind(state->sockfd, (struct sockaddr *)&state->local_addr, - conn->ip_addr->ai_addrlen); + int rc = bind(state->sockfd, (struct sockaddr *)&state->local_addr, + conn->ip_addr->ai_addrlen); if(rc) { failf(conn->data, "bind() failed; %s", Curl_strerror(conn, SOCKERRNO)); @@ -1148,8 +1147,11 @@ static CURLcode tftp_receive_packet(struct connectdata *conn) case TFTP_EVENT_ERROR: { unsigned short error = getrpacketblock(&state->rpacket); + char *str = (char *)state->rpacket.data + 4; + size_t strn = state->rbytes - 4; state->error = (tftp_error_t)error; - infof(data, "%s\n", (const char *)state->rpacket.data + 4); + if(Curl_strnlen(str, strn) < strn) + infof(data, "TFTP error: %s\n", str); break; } case TFTP_EVENT_ACK: @@ -1221,7 +1223,6 @@ static long tftp_state_timeout(struct connectdata *conn, tftp_event_t *event) **********************************************************/ static CURLcode tftp_multi_statemach(struct connectdata *conn, bool *done) { - int rc; tftp_event_t event; CURLcode result = CURLE_OK; struct Curl_easy *data = conn->data; @@ -1245,7 +1246,7 @@ static CURLcode tftp_multi_statemach(struct connectdata *conn, bool *done) } else { /* no timeouts to handle, check our socket */ - rc = SOCKET_READABLE(state->sockfd, 0); + int rc = SOCKET_READABLE(state->sockfd, 0); if(rc == -1) { /* bail out */ @@ -1368,7 +1369,6 @@ static CURLcode tftp_setup_connection(struct connectdata * conn) { struct Curl_easy *data = conn->data; char *type; - char command; conn->socktype = SOCK_DGRAM; /* UDP datagram based */ @@ -1380,6 +1380,7 @@ static CURLcode tftp_setup_connection(struct connectdata * conn) type = strstr(conn->host.rawalloc, ";mode="); if(type) { + char command; *type = 0; /* it was in the middle of the hostname */ command = Curl_raw_toupper(type[6]); diff --git a/Utilities/cmcurl/lib/transfer.c b/Utilities/cmcurl/lib/transfer.c index 131f2dc..7159d5c 100644 --- a/Utilities/cmcurl/lib/transfer.c +++ b/Utilities/cmcurl/lib/transfer.c @@ -106,15 +106,26 @@ char *Curl_checkheaders(const struct connectdata *conn, } #endif +CURLcode Curl_get_upload_buffer(struct Curl_easy *data) +{ + if(!data->state.ulbuf) { + data->state.ulbuf = malloc(data->set.upload_buffer_size); + if(!data->state.ulbuf) + return CURLE_OUT_OF_MEMORY; + } + return CURLE_OK; +} + /* * This function will call the read callback to fill our buffer with data * to upload. */ -CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) +CURLcode Curl_fillreadbuffer(struct connectdata *conn, size_t bytes, + size_t *nreadp) { struct Curl_easy *data = conn->data; - size_t buffersize = (size_t)bytes; - int nread; + size_t buffersize = bytes; + size_t nread; #ifdef CURL_DOES_CONVERSIONS bool sending_http_headers = FALSE; @@ -134,11 +145,9 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) data->req.upload_fromhere += (8 + 2); /* 32bit hex + CRLF */ } - /* this function returns a size_t, so we typecast to int to prevent warnings - with picky compilers */ Curl_set_in_callback(data, true); - nread = (int)data->state.fread_func(data->req.upload_fromhere, 1, - buffersize, data->state.in); + nread = data->state.fread_func(data->req.upload_fromhere, 1, + buffersize, data->state.in); Curl_set_in_callback(data, false); if(nread == CURL_READFUNC_ABORT) { @@ -167,7 +176,7 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) return CURLE_OK; /* nothing was read */ } - else if((size_t)nread > buffersize) { + else if(nread > buffersize) { /* the read function returned a too large value */ *nreadp = 0; failf(data, "read function returned funny value"); @@ -226,13 +235,13 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) #ifdef CURL_DOES_CONVERSIONS { CURLcode result; - int length; + size_t length; if(data->set.prefer_ascii) /* translate the protocol and data */ length = nread; else /* just translate the protocol portion */ - length = (int)strlen(hexbuffer); + length = strlen(hexbuffer); result = Curl_convert_to_network(data, data->req.upload_fromhere, length); /* Curl_convert_to_network calls failf if unsuccessful */ @@ -247,7 +256,7 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) infof(data, "Signaling end of chunked upload via terminating chunk.\n"); } - nread += (int)strlen(endofline_native); /* for the added end of line */ + nread += strlen(endofline_native); /* for the added end of line */ } #ifdef CURL_DOES_CONVERSIONS else if((data->set.prefer_ascii) && (!sending_http_headers)) { @@ -444,7 +453,6 @@ static CURLcode readwrite_data(struct Curl_easy *data, CURLcode result = CURLE_OK; ssize_t nread; /* number of bytes read */ size_t excess = 0; /* excess bytes read */ - bool is_empty_data = FALSE; bool readmore = FALSE; /* used by RTP to signal for more data */ int maxloops = 100; @@ -454,6 +462,7 @@ static CURLcode readwrite_data(struct Curl_easy *data, /* This is where we loop until we have read everything there is to read or we get a CURLE_AGAIN */ do { + bool is_empty_data = FALSE; size_t buffersize = data->set.buffer_size; size_t bytestoread = buffersize; @@ -660,14 +669,14 @@ static CURLcode readwrite_data(struct Curl_easy *data, if(data->set.verbose) { if(k->badheader) { Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff, - (size_t)k->hbuflen, conn); + (size_t)k->hbuflen); if(k->badheader == HEADER_PARTHEADER) Curl_debug(data, CURLINFO_DATA_IN, - k->str, (size_t)nread, conn); + k->str, (size_t)nread); } else Curl_debug(data, CURLINFO_DATA_IN, - k->str, (size_t)nread, conn); + k->str, (size_t)nread); } #ifndef CURL_DISABLE_HTTP @@ -797,7 +806,7 @@ static CURLcode readwrite_data(struct Curl_easy *data, nread); } } - else + else if(!k->ignorebody) result = Curl_unencode_write(conn, k->writer_stack, k->str, nread); } k->badheader = HEADER_NORMAL; /* taken care of now */ @@ -869,6 +878,26 @@ static CURLcode done_sending(struct connectdata *conn, return CURLE_OK; } +#ifdef WIN32 +#ifndef SIO_IDEAL_SEND_BACKLOG_QUERY +#define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747B +#endif + +static void win_update_buffer_size(curl_socket_t sockfd) +{ + int result; + ULONG ideal; + DWORD ideallen; + result = WSAIoctl(sockfd, SIO_IDEAL_SEND_BACKLOG_QUERY, 0, 0, + &ideal, sizeof(ideal), &ideallen, 0, 0); + if(result == 0) { + setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, + (const char *)&ideal, sizeof(ideal)); + } +} +#else +#define win_update_buffer_size(x) +#endif /* * Send data to upload to the server, when the socket is writable. @@ -894,13 +923,16 @@ static CURLcode readwrite_upload(struct Curl_easy *data, /* only read more data if there's no upload data already present in the upload buffer */ if(0 == k->upload_present) { + result = Curl_get_upload_buffer(data); + if(result) + return result; /* init the "upload from here" pointer */ - k->upload_fromhere = data->state.uploadbuffer; + k->upload_fromhere = data->state.ulbuf; if(!k->upload_done) { /* HTTP pollution, this should be written nicer to become more protocol agnostic. */ - int fillcount; + size_t fillcount; struct HTTP *http = k->protop; if((k->exp100 == EXP100_SENDING_REQUEST) && @@ -931,7 +963,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data, if(result) return result; - nread = (ssize_t)fillcount; + nread = fillcount; } else nread = 0; /* we're done uploading/reading */ @@ -959,7 +991,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data, (data->set.crlf))) { /* Do we need to allocate a scratch buffer? */ if(!data->state.scratch) { - data->state.scratch = malloc(2 * data->set.buffer_size); + data->state.scratch = malloc(2 * UPLOAD_BUFSIZE); if(!data->state.scratch) { failf(data, "Failed to alloc scratch buffer!"); @@ -1020,14 +1052,15 @@ static CURLcode readwrite_upload(struct Curl_easy *data, k->upload_fromhere, /* buffer pointer */ k->upload_present, /* buffer size */ &bytes_written); /* actually sent */ - if(result) return result; + win_update_buffer_size(conn->writesockfd); + if(data->set.verbose) /* show the data before we change the pointer upload_fromhere */ Curl_debug(data, CURLINFO_DATA_OUT, k->upload_fromhere, - (size_t)bytes_written, conn); + (size_t)bytes_written); k->writebytecount += bytes_written; @@ -1050,7 +1083,10 @@ static CURLcode readwrite_upload(struct Curl_easy *data, } else { /* we've uploaded that buffer now */ - k->upload_fromhere = data->state.uploadbuffer; + result = Curl_get_upload_buffer(data); + if(result) + return result; + k->upload_fromhere = data->state.ulbuf; k->upload_present = 0; /* no more bytes left */ if(k->upload_done) { @@ -1482,7 +1518,7 @@ static size_t strlen_url(const char *url, bool relative) switch(*ptr) { case '?': left = FALSE; - /* fall through */ + /* FALLTHROUGH */ default: if(urlchar_needs_escaping(*ptr)) newlen += 2; @@ -1527,7 +1563,7 @@ static void strcpy_url(char *output, const char *url, bool relative) switch(*iptr) { case '?': left = FALSE; - /* fall through */ + /* FALLTHROUGH */ default: if(urlchar_needs_escaping(*iptr)) { snprintf(optr, 4, "%%%02x", *iptr); diff --git a/Utilities/cmcurl/lib/transfer.h b/Utilities/cmcurl/lib/transfer.h index 9ba398d..9263e5b 100644 --- a/Utilities/cmcurl/lib/transfer.h +++ b/Utilities/cmcurl/lib/transfer.h @@ -51,9 +51,11 @@ int Curl_single_getsock(const struct connectdata *conn, curl_socket_t *socks, int numsocks); CURLcode Curl_readrewind(struct connectdata *conn); -CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp); +CURLcode Curl_fillreadbuffer(struct connectdata *conn, size_t bytes, + size_t *nreadp); CURLcode Curl_retry_request(struct connectdata *conn, char **url); bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc); +CURLcode Curl_get_upload_buffer(struct Curl_easy *data); /* This sets up a forthcoming transfer */ void diff --git a/Utilities/cmcurl/lib/url.c b/Utilities/cmcurl/lib/url.c index 701f83a..f159008 100644 --- a/Utilities/cmcurl/lib/url.c +++ b/Utilities/cmcurl/lib/url.c @@ -127,7 +127,6 @@ bool curl_win32_idn_to_ascii(const char *in, char **out); static void conn_free(struct connectdata *conn); static void free_fixed_hostname(struct hostname *host); -static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke); static CURLcode parse_url_login(struct Curl_easy *data, struct connectdata *conn, char **userptr, char **passwdptr, @@ -368,11 +367,9 @@ CURLcode Curl_close(struct Curl_easy *data) Curl_safefree(data->state.buffer); Curl_safefree(data->state.headerbuff); - + Curl_safefree(data->state.ulbuf); Curl_flush_cookies(data, 1); - Curl_digest_cleanup(data); - Curl_safefree(data->info.contenttype); Curl_safefree(data->info.wouldredirect); @@ -535,6 +532,7 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data) set->expect_100_timeout = 1000L; /* Wait for a second by default. */ set->sep_headers = TRUE; /* separated header lists by default */ set->buffer_size = READBUFFER_SIZE; + set->upload_buffer_size = UPLOAD_BUFSIZE; set->happy_eyeballs_timeout = CURL_HET_DEFAULT; Curl_http2_init_userset(set); @@ -734,20 +732,23 @@ static void conn_free(struct connectdata *conn) * primary connection, like when freeing room in the connection cache or * killing of a dead old connection. * + * A connection needs an easy handle when closing down. We support this passed + * in separately since the connection to get closed here is often already + * disassociated from an easy handle. + * * This function MUST NOT reset state in the Curl_easy struct if that * isn't strictly bound to the life-time of *this* particular connection. * */ -CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection) +CURLcode Curl_disconnect(struct Curl_easy *data, + struct connectdata *conn, bool dead_connection) { - struct Curl_easy *data; if(!conn) return CURLE_OK; /* this is closed and fine already */ - data = conn->data; if(!data) { - DEBUGF(fprintf(stderr, "DISCONNECT without easy handle, ignoring\n")); + DEBUGF(infof(data, "DISCONNECT without easy handle, ignoring\n")); return CURLE_OK; } @@ -755,13 +756,12 @@ CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection) * If this connection isn't marked to force-close, leave it open if there * are other users of it */ - if(!conn->bits.close && - (conn->send_pipe.size + conn->recv_pipe.size)) { - DEBUGF(infof(data, "Curl_disconnect, usecounter: %d\n", - conn->send_pipe.size + conn->recv_pipe.size)); + if(CONN_INUSE(conn) && !dead_connection) { + DEBUGF(infof(data, "Curl_disconnect when inuse: %zu\n", CONN_INUSE(conn))); return CURLE_OK; } + conn->data = data; if(conn->dns_entry != NULL) { Curl_resolv_unlock(data, conn->dns_entry); conn->dns_entry = NULL; @@ -787,16 +787,12 @@ CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection) free_fixed_hostname(&conn->http_proxy.host); free_fixed_hostname(&conn->socks_proxy.host); + DEBUGASSERT(conn->data == data); + /* this assumes that the pointer is still there after the connection was + detected from the cache */ Curl_ssl_close(conn, FIRSTSOCKET); - /* Indicate to all handles on the pipe that we're dead */ - if(Curl_pipeline_wanted(data->multi, CURLPIPE_ANY)) { - signalPipeClose(&conn->send_pipe, TRUE); - signalPipeClose(&conn->recv_pipe, TRUE); - } - conn_free(conn); - return CURLE_OK; } @@ -848,6 +844,7 @@ static int IsPipeliningPossible(const struct Curl_easy *handle, return avail; } +/* Returns non-zero if a handle was removed */ int Curl_removeHandleFromPipeline(struct Curl_easy *handle, struct curl_llist *pipeline) { @@ -884,6 +881,16 @@ static void Curl_printPipeline(struct curl_llist *pipeline) static struct Curl_easy* gethandleathead(struct curl_llist *pipeline) { struct curl_llist_element *curr = pipeline->head; +#ifdef DEBUGBUILD + { + struct curl_llist_element *p = pipeline->head; + while(p) { + struct Curl_easy *e = p->ptr; + DEBUGASSERT(GOOD_EASY_HANDLE(e)); + p = p->next; + } + } +#endif if(curr) { return (struct Curl_easy *) curr->ptr; } @@ -896,41 +903,22 @@ static struct Curl_easy* gethandleathead(struct curl_llist *pipeline) void Curl_getoff_all_pipelines(struct Curl_easy *data, struct connectdata *conn) { - bool recv_head = (conn->readchannel_inuse && - Curl_recvpipe_head(data, conn)); - bool send_head = (conn->writechannel_inuse && - Curl_sendpipe_head(data, conn)); - - if(Curl_removeHandleFromPipeline(data, &conn->recv_pipe) && recv_head) - Curl_pipeline_leave_read(conn); - if(Curl_removeHandleFromPipeline(data, &conn->send_pipe) && send_head) - Curl_pipeline_leave_write(conn); -} - -static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke) -{ - struct curl_llist_element *curr; - - if(!pipeline) + if(!conn->bundle) return; + if(conn->bundle->multiuse == BUNDLE_PIPELINING) { + bool recv_head = (conn->readchannel_inuse && + Curl_recvpipe_head(data, conn)); + bool send_head = (conn->writechannel_inuse && + Curl_sendpipe_head(data, conn)); - curr = pipeline->head; - while(curr) { - struct curl_llist_element *next = curr->next; - struct Curl_easy *data = (struct Curl_easy *) curr->ptr; - -#ifdef DEBUGBUILD /* debug-only code */ - if(data->magic != CURLEASY_MAGIC_NUMBER) { - /* MAJOR BADNESS */ - infof(data, "signalPipeClose() found BAAD easy handle\n"); - } -#endif - - if(pipe_broke) - data->state.pipe_broke = TRUE; - Curl_multi_handlePipeBreak(data); - Curl_llist_remove(pipeline, curr, NULL); - curr = next; + if(Curl_removeHandleFromPipeline(data, &conn->recv_pipe) && recv_head) + Curl_pipeline_leave_read(conn); + if(Curl_removeHandleFromPipeline(data, &conn->send_pipe) && send_head) + Curl_pipeline_leave_write(conn); + } + else { + (void)Curl_removeHandleFromPipeline(data, &conn->recv_pipe); + (void)Curl_removeHandleFromPipeline(data, &conn->send_pipe); } } @@ -959,12 +947,13 @@ static bool extract_if_dead(struct connectdata *conn, struct Curl_easy *data) { size_t pipeLen = conn->send_pipe.size + conn->recv_pipe.size; - if(!pipeLen && !conn->inuse) { + if(!pipeLen && !CONN_INUSE(conn)) { /* The check for a dead socket makes sense only if there are no handles in pipeline and the connection isn't already marked in use */ bool dead; + conn->data = data; if(conn->handler->connection_check) { /* The protocol has a special method for checking the state of the connection. Use it to check if the connection is dead. */ @@ -979,9 +968,9 @@ static bool extract_if_dead(struct connectdata *conn, } if(dead) { - conn->data = data; infof(data, "Connection %ld seems to be dead!\n", conn->connection_id); Curl_conncache_remove_conn(conn, FALSE); + conn->data = NULL; /* detach */ return TRUE; } } @@ -1025,7 +1014,7 @@ static void prune_dead_connections(struct Curl_easy *data) while(Curl_conncache_foreach(data, data->state.conn_cache, &prune, call_extract_if_dead)) { /* disconnect it */ - (void)Curl_disconnect(prune.extracted, /* dead_connection */TRUE); + (void)Curl_disconnect(data, prune.extracted, /* dead_connection */TRUE); } data->state.conn_cache->last_cleanup = now; } @@ -1139,7 +1128,7 @@ ConnectionExists(struct Curl_easy *data, if(extract_if_dead(check, data)) { /* disconnect it */ - (void)Curl_disconnect(check, /* dead_connection */TRUE); + (void)Curl_disconnect(data, check, /* dead_connection */TRUE); continue; } @@ -1267,12 +1256,12 @@ ConnectionExists(struct Curl_easy *data, } } - if(!canpipe && check->inuse) + if(!canpipe && CONN_INUSE(check)) /* this request can't be pipelined but the checked connection is already in use so we skip it */ continue; - if((check->inuse) && (check->data->multi != needle->data->multi)) + if(CONN_INUSE(check) && (check->data->multi != needle->data->multi)) /* this could be subject for pipeline/multiplex use, but only if they belong to the same multi handle */ continue; @@ -1464,7 +1453,7 @@ ConnectionExists(struct Curl_easy *data, if(chosen) { /* mark it as used before releasing the lock */ - chosen->inuse = TRUE; + chosen->data = data; /* own it! */ Curl_conncache_unlock(needle); *usethis = chosen; return TRUE; /* yes, we found one to use! */ @@ -1554,7 +1543,11 @@ int Curl_protocol_getsock(struct connectdata *conn, { if(conn->handler->proto_getsock) return conn->handler->proto_getsock(conn, socks, numsocks); - return GETSOCK_BLANK; + /* Backup getsock logic. Since there is a live socket in use, we must wait + for it or it will be removed from watching when the multi_socket API is + used. */ + socks[0] = conn->sock[FIRSTSOCKET]; + return GETSOCK_READSOCK(0) | GETSOCK_WRITESOCK(0); } int Curl_doing_getsock(struct connectdata *conn, @@ -1999,7 +1992,6 @@ static CURLcode parseurlandfillconn(struct Curl_easy *data, char *fragment; char *path = data->state.path; char *query; - int i; int rc; const char *protop = ""; CURLcode result; @@ -2051,6 +2043,7 @@ static CURLcode parseurlandfillconn(struct Curl_easy *data, ; /* do nothing */ } else { /* check for a scheme */ + int i; for(i = 0; i < 16 && data->change.url[i]; ++i) { if(data->change.url[i] == '/') break; @@ -2203,7 +2196,7 @@ static CURLcode parseurlandfillconn(struct Curl_easy *data, size_t s = strlen(slashbuf); protop = protobuf; if(s != 2) { - infof(data, "Unwillingly accepted illegal URL using %d slash%s!\n", + infof(data, "Unwillingly accepted illegal URL using %zu slash%s!\n", s, s>1?"es":""); if(data->change.url_alloc) @@ -2449,7 +2442,7 @@ static CURLcode setup_range(struct Curl_easy *data) free(s->range); if(s->resume_from) - s->range = aprintf("%" CURL_FORMAT_CURL_OFF_TU "-", s->resume_from); + s->range = aprintf("%" CURL_FORMAT_CURL_OFF_T "-", s->resume_from); else s->range = strdup(data->set.str[STRING_SET_RANGE]); @@ -2481,18 +2474,6 @@ static CURLcode setup_connection_internals(struct connectdata *conn) { const struct Curl_handler * p; CURLcode result; - struct Curl_easy *data = conn->data; - - /* in some case in the multi state-machine, we go back to the CONNECT state - and then a second (or third or...) call to this function will be made - without doing a DISCONNECT or DONE in between (since the connection is - yet in place) and therefore this function needs to first make sure - there's no lingering previous data allocated. */ - Curl_free_request_state(data); - - memset(&data->req, 0, sizeof(struct SingleRequest)); - data->req.maxdownload = -1; - conn->socktype = SOCK_STREAM; /* most of them are TCP streams */ /* Perform setup complement if some. */ @@ -2539,14 +2520,13 @@ static bool check_noproxy(const char *name, const char *no_proxy) * not be proxied, or an asterisk to override * all proxy variables) */ - size_t tok_start; - size_t tok_end; - const char *separator = ", "; - size_t no_proxy_len; - size_t namelen; - char *endptr; - if(no_proxy && no_proxy[0]) { + size_t tok_start; + size_t tok_end; + const char *separator = ", "; + size_t no_proxy_len; + size_t namelen; + char *endptr; if(strcasecompare("*", no_proxy)) { return TRUE; } @@ -3166,6 +3146,13 @@ static CURLcode parse_url_login(struct Curl_easy *data, if(userp) { char *newname; + if(data->set.disallow_username_in_url) { + failf(data, "Option DISALLOW_USERNAME_IN_URL is set " + "and url contains username."); + result = CURLE_LOGIN_DENIED; + goto out; + } + /* We have a user in the URL */ conn->bits.userpwd_in_url = TRUE; conn->bits.user_passwd = TRUE; /* enable user+password */ @@ -3982,6 +3969,7 @@ static void reuse_conn(struct connectdata *old_conn, Curl_safefree(old_conn->user); Curl_safefree(old_conn->passwd); + Curl_safefree(old_conn->options); Curl_safefree(old_conn->http_proxy.user); Curl_safefree(old_conn->socks_proxy.user); Curl_safefree(old_conn->http_proxy.passwd); @@ -4297,7 +4285,9 @@ static CURLcode create_conn(struct Curl_easy *data, conn->data = data; conn->bits.tcpconnect[FIRSTSOCKET] = TRUE; /* we are "connected */ - Curl_conncache_add_conn(data->state.conn_cache, conn); + result = Curl_conncache_add_conn(data->state.conn_cache, conn); + if(result) + goto out; /* * Setup whatever necessary for a resumed transfer @@ -4343,6 +4333,10 @@ static CURLcode create_conn(struct Curl_easy *data, data->set.str[STRING_SSL_CIPHER_LIST_ORIG]; data->set.proxy_ssl.primary.cipher_list = data->set.str[STRING_SSL_CIPHER_LIST_PROXY]; + data->set.ssl.primary.cipher_list13 = + data->set.str[STRING_SSL_CIPHER13_LIST_ORIG]; + data->set.proxy_ssl.primary.cipher_list13 = + data->set.str[STRING_SSL_CIPHER13_LIST_PROXY]; data->set.ssl.CRLfile = data->set.str[STRING_SSL_CRLFILE_ORIG]; data->set.proxy_ssl.CRLfile = data->set.str[STRING_SSL_CRLFILE_PROXY]; @@ -4468,13 +4462,11 @@ static CURLcode create_conn(struct Curl_easy *data, conn_candidate = Curl_conncache_extract_bundle(data, bundle); Curl_conncache_unlock(conn); - if(conn_candidate) { - /* Set the connection's owner correctly, then kill it */ - conn_candidate->data = data; - (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE); - } + if(conn_candidate) + (void)Curl_disconnect(data, conn_candidate, + /* dead_connection */ FALSE); else { - infof(data, "No more connections allowed to host: %d\n", + infof(data, "No more connections allowed to host: %zu\n", max_host_connections); connections_available = FALSE; } @@ -4491,12 +4483,9 @@ static CURLcode create_conn(struct Curl_easy *data, /* The cache is full. Let's see if we can kill a connection. */ conn_candidate = Curl_conncache_extract_oldest(data); - - if(conn_candidate) { - /* Set the connection's owner correctly, then kill it */ - conn_candidate->data = data; - (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE); - } + if(conn_candidate) + (void)Curl_disconnect(data, conn_candidate, + /* dead_connection */ FALSE); else { infof(data, "No connections available in cache\n"); connections_available = FALSE; @@ -4513,14 +4502,13 @@ static CURLcode create_conn(struct Curl_easy *data, goto out; } else { - /* Mark the connection as used, before we add it */ - conn->inuse = TRUE; - /* * This is a brand new connection, so let's store it in the connection * cache of ours! */ - Curl_conncache_add_conn(data->state.conn_cache, conn); + result = Curl_conncache_add_conn(data->state.conn_cache, conn); + if(result) + goto out; } #if defined(USE_NTLM) @@ -4656,12 +4644,16 @@ CURLcode Curl_connect(struct Curl_easy *data, *asyncp = FALSE; /* assume synchronous resolves by default */ + /* init the single-transfer specific data */ + Curl_free_request_state(data); + memset(&data->req, 0, sizeof(struct SingleRequest)); + data->req.maxdownload = -1; + /* call the stuff that needs to be called */ result = create_conn(data, in_connect, asyncp); if(!result) { - /* no error */ - if((*in_connect)->send_pipe.size || (*in_connect)->recv_pipe.size) + if(CONN_INUSE(*in_connect)) /* pipelining */ *protocol_done = TRUE; else if(!*asyncp) { @@ -4676,12 +4668,11 @@ CURLcode Curl_connect(struct Curl_easy *data, *in_connect = NULL; return result; } - - if(result && *in_connect) { - /* We're not allowed to return failure with memory left allocated - in the connectdata struct, free those here */ - Curl_disconnect(*in_connect, FALSE); /* close the connection */ - *in_connect = NULL; /* return a NULL */ + else if(result && *in_connect) { + /* We're not allowed to return failure with memory left allocated in the + connectdata struct, free those here */ + Curl_disconnect(data, *in_connect, TRUE); + *in_connect = NULL; /* return a NULL */ } return result; @@ -4701,16 +4692,18 @@ CURLcode Curl_init_do(struct Curl_easy *data, struct connectdata *conn) { struct SingleRequest *k = &data->req; - conn->bits.do_more = FALSE; /* by default there's no curl_do_more() to - use */ + if(conn) { + conn->bits.do_more = FALSE; /* by default there's no curl_do_more() to + use */ + /* if the protocol used doesn't support wildcards, switch it off */ + if(data->state.wildcardmatch && + !(conn->handler->flags & PROTOPT_WILDCARD)) + data->state.wildcardmatch = FALSE; + } data->state.done = FALSE; /* *_done() is not called yet */ data->state.expect100header = FALSE; - /* if the protocol used doesn't support wildcards, switch it off */ - if(data->state.wildcardmatch && - !(conn->handler->flags & PROTOPT_WILDCARD)) - data->state.wildcardmatch = FALSE; if(data->set.opt_no_body) /* in HTTP lingo, no body means using the HEAD request... */ diff --git a/Utilities/cmcurl/lib/url.h b/Utilities/cmcurl/lib/url.h index a70bd54..ef3ebf0 100644 --- a/Utilities/cmcurl/lib/url.h +++ b/Utilities/cmcurl/lib/url.h @@ -39,7 +39,8 @@ void Curl_freeset(struct Curl_easy * data); CURLcode Curl_close(struct Curl_easy *data); /* opposite of curl_open() */ CURLcode Curl_connect(struct Curl_easy *, struct connectdata **, bool *async, bool *protocol_connect); -CURLcode Curl_disconnect(struct connectdata *, bool dead_connection); +CURLcode Curl_disconnect(struct Curl_easy *data, + struct connectdata *, bool dead_connection); CURLcode Curl_protocol_connect(struct connectdata *conn, bool *done); CURLcode Curl_protocol_connecting(struct connectdata *conn, bool *done); CURLcode Curl_protocol_doing(struct connectdata *conn, bool *done); diff --git a/Utilities/cmcurl/lib/urldata.h b/Utilities/cmcurl/lib/urldata.h index 7fae00f..67db3b2 100644 --- a/Utilities/cmcurl/lib/urldata.h +++ b/Utilities/cmcurl/lib/urldata.h @@ -80,6 +80,7 @@ #define RESP_TIMEOUT (1800*1000) #include "cookie.h" +#include "psl.h" #include "formdata.h" #ifdef HAVE_NETINET_IN_H @@ -142,8 +143,13 @@ typedef ssize_t (Curl_recv)(struct connectdata *conn, /* connection data */ #endif /* HAVE_LIBSSH2_H */ /* The upload buffer size, should not be smaller than CURL_MAX_WRITE_SIZE, as - it needs to hold a full buffer as could be sent in a write callback */ -#define UPLOAD_BUFSIZE CURL_MAX_WRITE_SIZE + it needs to hold a full buffer as could be sent in a write callback. + + The size was 16KB for many years but was bumped to 64KB because it makes + libcurl able to do significantly faster uploads in some circumstances. Even + larger buffers can help further, but this is deemed a fair memory/speed + compromise. */ +#define UPLOAD_BUFSIZE 65536 /* The "master buffer" is for HTTP pipelining */ #define MASTERBUF_SIZE 16384 @@ -156,11 +162,6 @@ typedef ssize_t (Curl_recv)(struct connectdata *conn, /* connection data */ #define GOOD_EASY_HANDLE(x) \ ((x) && ((x)->magic == CURLEASY_MAGIC_NUMBER)) -/* Some convenience macros to get the larger/smaller value out of two given. - We prefix with CURL to prevent name collisions. */ -#define CURLMAX(x,y) ((x)>(y)?(x):(y)) -#define CURLMIN(x,y) ((x)<(y)?(x):(y)) - #ifdef HAVE_GSSAPI /* Types needed for krb5-ftp connections */ struct krb5buffer { @@ -226,6 +227,7 @@ struct ssl_primary_config { char *random_file; /* path to file containing "random" data */ char *egdsocket; /* path to file containing the EGD daemon socket */ char *cipher_list; /* list of ciphers to use */ + char *cipher_list13; /* list of TLS 1.3 cipher suites to use */ }; struct ssl_config_data { @@ -779,11 +781,12 @@ struct connectdata { curl_closesocket_callback fclosesocket; /* function closing the socket(s) */ void *closesocket_client; - bool inuse; /* This is a marker for the connection cache logic. If this is - TRUE this handle is being used by one or more easy handles - and can only used by any other easy handle without careful - consideration (== only for pipelining/multiplexing) and it - cannot be used by another multi handle! */ + /* This is used by the connection cache logic. If this returns TRUE, this + handle is being used by one or more easy handles and can only used by any + other easy handle without careful consideration (== only for + pipelining/multiplexing) and it cannot be used by another multi + handle! */ +#define CONN_INUSE(c) ((c)->send_pipe.size + (c)->recv_pipe.size) /**** Fields set when inited and not modified again */ long connection_id; /* Contains a unique number to make it easier to @@ -1222,7 +1225,7 @@ struct UrlState { size_t headersize; /* size of the allocation */ char *buffer; /* download buffer */ - char uploadbuffer[UPLOAD_BUFSIZE + 1]; /* upload buffer */ + char *ulbuf; /* alloced upload buffer or NULL */ curl_off_t current_speed; /* the ProgressShow() function sets this, bytes / second */ bool this_is_a_follow; /* this is a followed Location: request */ @@ -1265,7 +1268,7 @@ struct UrlState { void *resolver; /* resolver state, if it is used in the URL state - ares_channel f.e. */ -#if defined(USE_OPENSSL) && defined(HAVE_OPENSSL_ENGINE_H) +#if defined(USE_OPENSSL) /* void instead of ENGINE to avoid bleeding OpenSSL into this header */ void *engine; #endif /* USE_OPENSSL */ @@ -1284,9 +1287,6 @@ struct UrlState { involved in this request */ bool expect100header; /* TRUE if we added Expect: 100-continue */ - bool pipe_broke; /* TRUE if the connection we were pipelined on broke - and we need to restart from the beginning */ - #if !defined(WIN32) && !defined(MSDOS) && !defined(__EMX__) && \ !defined(__SYMBIAN32__) /* do FTP line-end conversions on most platforms */ @@ -1346,7 +1346,7 @@ struct DynamicStatic { char *url; /* work URL, copied from UserDefined */ bool url_alloc; /* URL string is malloc()'ed */ char *referer; /* referer string */ - bool referer_alloc; /* referer sting is malloc()ed */ + bool referer_alloc; /* referer string is malloc()ed */ struct curl_slist *cookielist; /* list of cookie files set by curl_easy_setopt(COOKIEFILE) calls */ struct curl_slist *resolve; /* set to point to the set.resolve list when @@ -1400,6 +1400,8 @@ enum dupstring { STRING_SSL_PINNEDPUBLICKEY_PROXY, /* public key file to verify proxy */ STRING_SSL_CIPHER_LIST_ORIG, /* list of ciphers to use */ STRING_SSL_CIPHER_LIST_PROXY, /* list of ciphers to use */ + STRING_SSL_CIPHER13_LIST_ORIG, /* list of TLS 1.3 ciphers to use */ + STRING_SSL_CIPHER13_LIST_PROXY, /* list of TLS 1.3 ciphers to use */ STRING_SSL_EGDSOCKET, /* path to file containing the EGD daemon socket */ STRING_SSL_RANDOM_FILE, /* path to file containing "random" data */ STRING_USERAGENT, /* User-Agent string */ @@ -1407,6 +1409,7 @@ enum dupstring { STRING_SSL_CRLFILE_PROXY, /* crl file to check certificate */ STRING_SSL_ISSUERCERT_ORIG, /* issuer cert file to check certificate */ STRING_SSL_ISSUERCERT_PROXY, /* issuer cert file to check certificate */ + STRING_SSL_ENGINE, /* name of ssl engine */ STRING_USERNAME, /* <username>, if used */ STRING_PASSWORD, /* <password>, if used */ STRING_OPTIONS, /* <options>, if used */ @@ -1559,6 +1562,8 @@ struct UserDefined { curl_proxytype proxytype; /* what kind of proxy that is in use */ long dns_cache_timeout; /* DNS cache timeout */ long buffer_size; /* size of receive buffer to use */ + long upload_buffer_size; /* size of upload buffer to use, + keep it >= CURL_MAX_WRITE_SIZE */ void *private_data; /* application-private data */ struct curl_slist *http200aliases; /* linked list of aliases for http200 */ @@ -1581,8 +1586,6 @@ struct UserDefined { /* Here follows boolean settings that define how to behave during this session. They are STATIC, set by libcurl users or at least initially and they don't change during operations. */ - - bool printhost; /* printing host name in debug info */ bool get_filetime; /* get the time and get of the remote file */ bool tunnel_thru_httpproxy; /* use CONNECT through a HTTP proxy */ bool prefer_ascii; /* ASCII rather than binary */ @@ -1676,7 +1679,7 @@ struct UserDefined { bool stream_depends_e; /* set or don't set the Exclusive bit */ int stream_weight; - bool haproxyprotocol; /* whether to send HAProxy PROXY protocol header */ + bool haproxyprotocol; /* whether to send HAProxy PROXY protocol v1 header */ struct Curl_http2_dep *stream_dependents; @@ -1685,6 +1688,7 @@ struct UserDefined { curl_resolver_start_callback resolver_start; /* optional callback called before resolver start */ void *resolver_start_client; /* pointer to pass to resolver start callback */ + bool disallow_username_in_url; /* disallow username in url */ }; struct Names { @@ -1736,6 +1740,9 @@ struct Curl_easy { struct to which this "belongs" when used by the easy interface */ struct Curl_share *share; /* Share, handles global variable mutexing */ +#ifdef USE_LIBPSL + struct PslCache *psl; /* The associated PSL cache. */ +#endif struct SingleRequest req; /* Request-specific data */ struct UserDefined set; /* values set by the libcurl user */ struct DynamicStatic change; /* possibly modified userdefined data */ diff --git a/Utilities/cmcurl/lib/vauth/digest.c b/Utilities/cmcurl/lib/vauth/digest.c index 131d9da..cc6f169 100644 --- a/Utilities/cmcurl/lib/vauth/digest.c +++ b/Utilities/cmcurl/lib/vauth/digest.c @@ -158,7 +158,7 @@ static void auth_digest_sha256_to_ascii(unsigned char *source, /* 32 bytes */ /* Perform quoted-string escaping as described in RFC2616 and its errata */ static char *auth_digest_string_quoted(const char *source) { - char *dest, *d; + char *dest; const char *s = source; size_t n = 1; /* null terminator */ @@ -173,8 +173,8 @@ static char *auth_digest_string_quoted(const char *source) dest = malloc(n); if(dest) { + char *d = dest; s = source; - d = dest; while(*s) { if(*s == '"' || *s == '\\') { *d++ = '\\'; @@ -696,7 +696,6 @@ static CURLcode _Curl_auth_create_digest_http_message( unsigned char ha1[65]; /* 64 digits and 1 zero byte */ unsigned char ha2[65]; /* 64 digits and 1 zero byte */ char userh[65]; - char cnoncebuf[33]; char *cnonce = NULL; size_t cnonce_sz = 0; char *userp_quoted; @@ -707,6 +706,7 @@ static CURLcode _Curl_auth_create_digest_http_message( digest->nc = 1; if(!digest->cnonce) { + char cnoncebuf[33]; result = Curl_rand_hex(data, (unsigned char *)cnoncebuf, sizeof(cnoncebuf)); if(result) diff --git a/Utilities/cmcurl/lib/version.c b/Utilities/cmcurl/lib/version.c index 5b0d05a..05c2cd8 100644 --- a/Utilities/cmcurl/lib/version.c +++ b/Utilities/cmcurl/lib/version.c @@ -445,7 +445,7 @@ curl_version_info_data *curl_version_info(CURLversion stamp) #ifdef HAVE_BROTLI version_info.brotli_ver_num = BrotliDecoderVersion(); - brotli_version(brotli_buffer, sizeof brotli_buffer); + brotli_version(brotli_buffer, sizeof(brotli_buffer)); version_info.brotli_version = brotli_buffer; #endif diff --git a/Utilities/cmcurl/lib/vtls/axtls.c b/Utilities/cmcurl/lib/vtls/axtls.c index 5ed898b..b262392 100644 --- a/Utilities/cmcurl/lib/vtls/axtls.c +++ b/Utilities/cmcurl/lib/vtls/axtls.c @@ -29,6 +29,11 @@ #include "curl_setup.h" #ifdef USE_AXTLS + +#error axTLS support has been disabled in curl due to doubts about quality, +#error user dedication and a lack of use/testing. We urge users to consider +#error using a more established TLS backend instead. + #include <axTLS/config.h> #include <axTLS/ssl.h> #include "axtls.h" diff --git a/Utilities/cmcurl/lib/vtls/cyassl.c b/Utilities/cmcurl/lib/vtls/cyassl.c index 20ce460..e10398a 100644 --- a/Utilities/cmcurl/lib/vtls/cyassl.c +++ b/Utilities/cmcurl/lib/vtls/cyassl.c @@ -569,7 +569,7 @@ cyassl_connect_step2(struct connectdata *conn, return CURLE_SSL_PINNEDPUBKEYNOTMATCH; } - memset(&x509_parsed, 0, sizeof x509_parsed); + memset(&x509_parsed, 0, sizeof(x509_parsed)); if(Curl_parseX509(&x509_parsed, x509_der, x509_der + x509_der_len)) return CURLE_SSL_PINNEDPUBKEYNOTMATCH; @@ -968,6 +968,8 @@ static CURLcode Curl_cyassl_random(struct Curl_easy *data, return CURLE_FAILED_INIT; if(RNG_GenerateBlock(&rng, entropy, (unsigned)length)) return CURLE_FAILED_INIT; + if(FreeRng(&rng)) + return CURLE_FAILED_INIT; return CURLE_OK; } diff --git a/Utilities/cmcurl/lib/vtls/darwinssl.c b/Utilities/cmcurl/lib/vtls/darwinssl.c index 45fe49d..1aea0dc 100644 --- a/Utilities/cmcurl/lib/vtls/darwinssl.c +++ b/Utilities/cmcurl/lib/vtls/darwinssl.c @@ -1252,14 +1252,13 @@ static CURLcode darwinssl_version_from_curl(SSLProtocol *darwinver, return CURLE_OK; case CURL_SSLVERSION_TLSv1_3: /* TLS 1.3 support first appeared in iOS 11 and macOS 10.13 */ -#if CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 - /* We can assume __builtin_available() will always work in the - 10.13/11.0 SDK: */ +#if (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && HAVE_BUILTIN_AVAILABLE == 1 if(__builtin_available(macOS 10.13, iOS 11.0, *)) { *darwinver = kTLSProtocol13; return CURLE_OK; } -#endif /* CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 */ +#endif /* (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && + HAVE_BUILTIN_AVAILABLE == 1 */ break; } return CURLE_SSL_CONNECT_ERROR; @@ -1278,7 +1277,7 @@ set_ssl_version_min_max(struct connectdata *conn, int sockindex) /* macOS 10.5-10.7 supported TLS 1.0 only. macOS 10.8 and later, and iOS 5 and later, added TLS 1.1 and 1.2. macOS 10.13 and later, and iOS 11 and later, added TLS 1.3. */ -#if CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 +#if (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && HAVE_BUILTIN_AVAILABLE == 1 if(__builtin_available(macOS 10.13, iOS 11.0, *)) { max_supported_version_by_os = CURL_SSLVERSION_MAX_TLSv1_3; } @@ -1287,7 +1286,8 @@ set_ssl_version_min_max(struct connectdata *conn, int sockindex) } #else max_supported_version_by_os = CURL_SSLVERSION_MAX_TLSv1_2; -#endif /* CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 */ +#endif /* (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && + HAVE_BUILTIN_AVAILABLE == 1 */ switch(ssl_version) { case CURL_SSLVERSION_DEFAULT: @@ -1430,7 +1430,7 @@ static CURLcode darwinssl_connect_step1(struct connectdata *conn, case CURL_SSLVERSION_DEFAULT: case CURL_SSLVERSION_TLSv1: (void)SSLSetProtocolVersionMin(BACKEND->ssl_ctx, kTLSProtocol1); -#if CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 +#if (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && HAVE_BUILTIN_AVAILABLE == 1 if(__builtin_available(macOS 10.13, iOS 11.0, *)) { (void)SSLSetProtocolVersionMax(BACKEND->ssl_ctx, kTLSProtocol13); } @@ -1439,7 +1439,8 @@ static CURLcode darwinssl_connect_step1(struct connectdata *conn, } #else (void)SSLSetProtocolVersionMax(BACKEND->ssl_ctx, kTLSProtocol12); -#endif /* CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 */ +#endif /* (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && + HAVE_BUILTIN_AVAILABLE == 1 */ break; case CURL_SSLVERSION_TLSv1_0: case CURL_SSLVERSION_TLSv1_1: @@ -1572,6 +1573,35 @@ static CURLcode darwinssl_connect_step1(struct connectdata *conn, } #endif /* CURL_BUILD_MAC_10_8 || CURL_BUILD_IOS */ +#if (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && HAVE_BUILTIN_AVAILABLE == 1 + if(conn->bits.tls_enable_alpn) { + if(__builtin_available(macOS 10.13.4, iOS 11, *)) { + CFMutableArrayRef alpnArr = CFArrayCreateMutable(NULL, 0, + &kCFTypeArrayCallBacks); + +#ifdef USE_NGHTTP2 + if(data->set.httpversion >= CURL_HTTP_VERSION_2 && + (!SSL_IS_PROXY() || !conn->bits.tunnel_proxy)) { + CFArrayAppendValue(alpnArr, CFSTR(NGHTTP2_PROTO_VERSION_ID)); + infof(data, "ALPN, offering %s\n", NGHTTP2_PROTO_VERSION_ID); + } +#endif + + CFArrayAppendValue(alpnArr, CFSTR(ALPN_HTTP_1_1)); + infof(data, "ALPN, offering %s\n", ALPN_HTTP_1_1); + + /* expects length prefixed preference ordered list of protocols in wire + * format + */ + err = SSLSetALPNProtocols(BACKEND->ssl_ctx, alpnArr); + if(err != noErr) + infof(data, "WARNING: failed to set ALPN protocols; OSStatus %d\n", + err); + CFRelease(alpnArr); + } + } +#endif + if(SSL_SET_OPTION(key)) { infof(data, "WARNING: SSL: CURLOPT_SSLKEY is ignored by Secure " "Transport. The private key must be in the Keychain.\n"); @@ -2466,6 +2496,39 @@ darwinssl_connect_step2(struct connectdata *conn, int sockindex) break; } +#if(CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && HAVE_BUILTIN_AVAILABLE == 1 + if(conn->bits.tls_enable_alpn) { + if(__builtin_available(macOS 10.13.4, iOS 11, *)) { + CFArrayRef alpnArr = NULL; + CFStringRef chosenProtocol = NULL; + err = SSLCopyALPNProtocols(BACKEND->ssl_ctx, &alpnArr); + + if(err == noErr && alpnArr && CFArrayGetCount(alpnArr) >= 1) + chosenProtocol = CFArrayGetValueAtIndex(alpnArr, 0); + +#ifdef USE_NGHTTP2 + if(chosenProtocol && + !CFStringCompare(chosenProtocol, CFSTR(NGHTTP2_PROTO_VERSION_ID), + 0)) { + conn->negnpn = CURL_HTTP_VERSION_2; + } + else +#endif + if(chosenProtocol && + !CFStringCompare(chosenProtocol, CFSTR(ALPN_HTTP_1_1), 0)) { + conn->negnpn = CURL_HTTP_VERSION_1_1; + } + else + infof(data, "ALPN, server did not agree to a protocol\n"); + + /* chosenProtocol is a reference to the string within alpnArr + and doesn't need to be freed separately */ + if(alpnArr) + CFRelease(alpnArr); + } + } +#endif + return CURLE_OK; } } diff --git a/Utilities/cmcurl/lib/vtls/gskit.c b/Utilities/cmcurl/lib/vtls/gskit.c index b0856cd..a0b4960 100644 --- a/Utilities/cmcurl/lib/vtls/gskit.c +++ b/Utilities/cmcurl/lib/vtls/gskit.c @@ -328,7 +328,7 @@ static CURLcode set_ciphers(struct connectdata *conn, GSKit tokens are always shorter than their cipher names, allocated buffers will always be large enough to accommodate the result. */ l = strlen(cipherlist) + 1; - memset((char *) ciphers, 0, sizeof ciphers); + memset((char *) ciphers, 0, sizeof(ciphers)); for(i = 0; i < CURL_GSKPROTO_LAST; i++) { ciphers[i].buf = malloc(l); if(!ciphers[i].buf) { @@ -536,18 +536,18 @@ inetsocketpair(int sv[2]) lfd = socket(AF_INET, SOCK_STREAM, 0); if(lfd < 0) return -1; - memset((char *) &addr1, 0, sizeof addr1); + memset((char *) &addr1, 0, sizeof(addr1)); addr1.sin_family = AF_INET; addr1.sin_addr.s_addr = htonl(INADDR_LOOPBACK); addr1.sin_port = 0; - if(bind(lfd, (struct sockaddr *) &addr1, sizeof addr1) || + if(bind(lfd, (struct sockaddr *) &addr1, sizeof(addr1)) || listen(lfd, 2) < 0) { close(lfd); return -1; } /* Get the allocated port. */ - len = sizeof addr1; + len = sizeof(addr1); if(getsockname(lfd, (struct sockaddr *) &addr1, &len) < 0) { close(lfd); return -1; @@ -562,7 +562,7 @@ inetsocketpair(int sv[2]) /* Request unblocking connection to the listening socket. */ curlx_nonblock(cfd, TRUE); - if(connect(cfd, (struct sockaddr *) &addr1, sizeof addr1) < 0 && + if(connect(cfd, (struct sockaddr *) &addr1, sizeof(addr1)) < 0 && errno != EINPROGRESS) { close(lfd); close(cfd); @@ -570,7 +570,7 @@ inetsocketpair(int sv[2]) } /* Get the client dynamic port for intrusion check below. */ - len = sizeof addr2; + len = sizeof(addr2); if(getsockname(cfd, (struct sockaddr *) &addr2, &len) < 0) { close(lfd); close(cfd); @@ -580,7 +580,7 @@ inetsocketpair(int sv[2]) /* Accept the incoming connection and get the server socket. */ curlx_nonblock(lfd, TRUE); for(;;) { - len = sizeof addr1; + len = sizeof(addr1); sfd = accept(lfd, (struct sockaddr *) &addr1, &len); if(sfd < 0) { close(lfd); @@ -644,7 +644,7 @@ static int pipe_ssloverssl(struct connectdata *conn, int sockindex, /* Try getting data from HTTPS proxy and pipe it upstream. */ n = 0; i = gsk_secure_soc_read(connproxyssl->backend->handle, - buf, sizeof buf, &n); + buf, sizeof(buf), &n); switch(i) { case GSK_OK: if(n) { @@ -665,7 +665,7 @@ static int pipe_ssloverssl(struct connectdata *conn, int sockindex, if(FD_ISSET(BACKEND->remotefd, &fds_read) && FD_ISSET(conn->sock[sockindex], &fds_write)) { /* Pipe data to HTTPS proxy. */ - n = read(BACKEND->remotefd, buf, sizeof buf); + n = read(BACKEND->remotefd, buf, sizeof(buf)); if(n < 0) return -1; if(n) { @@ -864,13 +864,13 @@ static CURLcode gskit_connect_step1(struct connectdata *conn, int sockindex) BACKEND->localfd = sockpair[0]; BACKEND->remotefd = sockpair[1]; setsockopt(BACKEND->localfd, SOL_SOCKET, SO_RCVBUF, - (void *) sobufsize, sizeof sobufsize); + (void *) sobufsize, sizeof(sobufsize)); setsockopt(BACKEND->remotefd, SOL_SOCKET, SO_RCVBUF, - (void *) sobufsize, sizeof sobufsize); + (void *) sobufsize, sizeof(sobufsize)); setsockopt(BACKEND->localfd, SOL_SOCKET, SO_SNDBUF, - (void *) sobufsize, sizeof sobufsize); + (void *) sobufsize, sizeof(sobufsize)); setsockopt(BACKEND->remotefd, SOL_SOCKET, SO_SNDBUF, - (void *) sobufsize, sizeof sobufsize); + (void *) sobufsize, sizeof(sobufsize)); curlx_nonblock(BACKEND->localfd, TRUE); curlx_nonblock(BACKEND->remotefd, TRUE); } @@ -977,7 +977,7 @@ static CURLcode gskit_connect_step1(struct connectdata *conn, int sockindex) if(!result) { /* Start handshake. Try asynchronous first. */ - memset(&commarea, 0, sizeof commarea); + memset(&commarea, 0, sizeof(commarea)); BACKEND->iocport = QsoCreateIOCompletionPort(); if(BACKEND->iocport != -1) { result = gskit_status(data, @@ -1333,11 +1333,11 @@ static int Curl_gskit_check_cxn(struct connectdata *cxn) return 0; /* connection has been closed */ err = 0; - errlen = sizeof err; + errlen = sizeof(err); if(getsockopt(cxn->sock[FIRSTSOCKET], SOL_SOCKET, SO_ERROR, (unsigned char *) &err, &errlen) || - errlen != sizeof err || err) + errlen != sizeof(err) || err) return 0; /* connection has been closed */ return -1; /* connection status unknown */ diff --git a/Utilities/cmcurl/lib/vtls/nss.c b/Utilities/cmcurl/lib/vtls/nss.c index 7cd450c..89f8183 100644 --- a/Utilities/cmcurl/lib/vtls/nss.c +++ b/Utilities/cmcurl/lib/vtls/nss.c @@ -394,7 +394,7 @@ static PK11SlotInfo* nss_find_slot_by_name(const char *slot_name) /* wrap 'ptr' as list node and tail-insert into 'list' */ static CURLcode insert_wrapped_ptr(struct curl_llist *list, void *ptr) { - struct ptr_list_wrap *wrap = malloc(sizeof *wrap); + struct ptr_list_wrap *wrap = malloc(sizeof(*wrap)); if(!wrap) return CURLE_OUT_OF_MEMORY; @@ -914,11 +914,11 @@ static CURLcode display_conn_info(struct connectdata *conn, PRFileDesc *sock) PRTime now; int i; - if(SSL_GetChannelInfo(sock, &channel, sizeof channel) == - SECSuccess && channel.length == sizeof channel && + if(SSL_GetChannelInfo(sock, &channel, sizeof(channel)) == + SECSuccess && channel.length == sizeof(channel) && channel.cipherSuite) { if(SSL_GetCipherSuiteInfo(channel.cipherSuite, - &suite, sizeof suite) == SECSuccess) { + &suite, sizeof(suite)) == SECSuccess) { infof(conn->data, "SSL connection using %s\n", suite.cipherSuiteName); } } @@ -1345,7 +1345,8 @@ static CURLcode nss_init(struct Curl_easy *data) return CURLE_OUT_OF_MEMORY; /* the default methods just call down to the lower I/O layer */ - memcpy(&nspr_io_methods, PR_GetDefaultIOMethods(), sizeof nspr_io_methods); + memcpy(&nspr_io_methods, PR_GetDefaultIOMethods(), + sizeof(nspr_io_methods)); /* override certain methods in the table by our wrappers */ nspr_io_methods.recv = nspr_io_recv; diff --git a/Utilities/cmcurl/lib/vtls/openssl.c b/Utilities/cmcurl/lib/vtls/openssl.c index f6a4bd3..a487f55 100644 --- a/Utilities/cmcurl/lib/vtls/openssl.c +++ b/Utilities/cmcurl/lib/vtls/openssl.c @@ -50,9 +50,6 @@ #include "hostcheck.h" #include "curl_printf.h" #include <openssl/ssl.h> -#ifdef HAVE_OPENSSL_ENGINE_H -#include <openssl/engine.h> -#endif #include <openssl/rand.h> #include <openssl/x509v3.h> #ifndef OPENSSL_NO_DSA @@ -72,6 +69,12 @@ #include <openssl/ocsp.h> #endif +#if (OPENSSL_VERSION_NUMBER >= 0x10000000L) && /* 1.0.0 or later */ \ + !defined(OPENSSL_NO_ENGINE) +#define USE_OPENSSL_ENGINE +#include <openssl/engine.h> +#endif + #include "warnless.h" #include "non-ascii.h" /* for Curl_convert_from_utf8 prototype */ @@ -83,7 +86,7 @@ #error "OPENSSL_VERSION_NUMBER not defined" #endif -#if defined(HAVE_OPENSSL_ENGINE_H) +#ifdef USE_OPENSSL_ENGINE #include <openssl/ui.h> #endif @@ -166,6 +169,17 @@ static unsigned long OpenSSL_version_num(void) #define HAVE_KEYLOG_CALLBACK #endif +/* Whether SSL_CTX_set_ciphersuites is available. + * OpenSSL: supported since 1.1.1 (commit a53b5be6a05) + * BoringSSL: no + * LibreSSL: no + */ +#if ((OPENSSL_VERSION_NUMBER >= 0x10101000L) && \ + !defined(LIBRESSL_VERSION_NUMBER) && \ + !defined(OPENSSL_IS_BORINGSSL)) +#define HAVE_SSL_CTX_SET_CIPHERSUITES +#endif + #if defined(LIBRESSL_VERSION_NUMBER) #define OSSL_PACKAGE "LibreSSL" #elif defined(OPENSSL_IS_BORINGSSL) @@ -504,7 +518,7 @@ static int do_file_type(const char *type) return -1; } -#if defined(HAVE_OPENSSL_ENGINE_H) +#ifdef USE_OPENSSL_ENGINE /* * Supply default password to the engine user interface conversation. * The password is passed by OpenSSL engine from ENGINE_load_private_key() @@ -544,8 +558,20 @@ static int ssl_ui_writer(UI *ui, UI_STRING *uis) } return (UI_method_get_writer(UI_OpenSSL()))(ui, uis); } + +/* + * Check if a given string is a PKCS#11 URI + */ +static bool is_pkcs11_uri(const char *string) +{ + return (string && strncasecompare(string, "pkcs11:", 7)); +} + #endif +static CURLcode Curl_ossl_set_engine(struct Curl_easy *data, + const char *engine); + static int cert_stuff(struct connectdata *conn, SSL_CTX* ctx, @@ -606,8 +632,18 @@ int cert_stuff(struct connectdata *conn, } break; case SSL_FILETYPE_ENGINE: -#if defined(HAVE_OPENSSL_ENGINE_H) && defined(ENGINE_CTRL_GET_CMD_FROM_NAME) +#if defined(USE_OPENSSL_ENGINE) && defined(ENGINE_CTRL_GET_CMD_FROM_NAME) { + /* Implicitly use pkcs11 engine if none was provided and the + * cert_file is a PKCS#11 URI */ + if(!data->state.engine) { + if(is_pkcs11_uri(cert_file)) { + if(Curl_ossl_set_engine(data, "pkcs11") != CURLE_OK) { + return 0; + } + } + } + if(data->state.engine) { const char *cmd_name = "LOAD_CERT_CTRL"; struct { @@ -781,9 +817,20 @@ int cert_stuff(struct connectdata *conn, } break; case SSL_FILETYPE_ENGINE: -#ifdef HAVE_OPENSSL_ENGINE_H +#ifdef USE_OPENSSL_ENGINE { /* XXXX still needs some work */ EVP_PKEY *priv_key = NULL; + + /* Implicitly use pkcs11 engine if none was provided and the + * key_file is a PKCS#11 URI */ + if(!data->state.engine) { + if(is_pkcs11_uri(key_file)) { + if(Curl_ossl_set_engine(data, "pkcs11") != CURLE_OK) { + return 0; + } + } + } + if(data->state.engine) { UI_METHOD *ui_method = UI_create_method((char *)"curl user interface"); @@ -1088,7 +1135,7 @@ static int Curl_ossl_check_cxn(struct connectdata *conn) static CURLcode Curl_ossl_set_engine(struct Curl_easy *data, const char *engine) { -#if defined(USE_OPENSSL) && defined(HAVE_OPENSSL_ENGINE_H) +#ifdef USE_OPENSSL_ENGINE ENGINE *e; #if OPENSSL_VERSION_NUMBER >= 0x00909000L @@ -1133,7 +1180,7 @@ static CURLcode Curl_ossl_set_engine(struct Curl_easy *data, */ static CURLcode Curl_ossl_set_engine_default(struct Curl_easy *data) { -#ifdef HAVE_OPENSSL_ENGINE_H +#ifdef USE_OPENSSL_ENGINE if(data->state.engine) { if(ENGINE_set_default(data->state.engine, ENGINE_METHOD_ALL) > 0) { infof(data, "set default crypto engine '%s'\n", @@ -1156,7 +1203,7 @@ static CURLcode Curl_ossl_set_engine_default(struct Curl_easy *data) static struct curl_slist *Curl_ossl_engines_list(struct Curl_easy *data) { struct curl_slist *list = NULL; -#if defined(USE_OPENSSL) && defined(HAVE_OPENSSL_ENGINE_H) +#ifdef USE_OPENSSL_ENGINE struct curl_slist *beg; ENGINE *e; @@ -1312,7 +1359,7 @@ static void Curl_ossl_session_free(void *ptr) */ static void Curl_ossl_close_all(struct Curl_easy *data) { -#ifdef HAVE_OPENSSL_ENGINE_H +#ifdef USE_OPENSSL_ENGINE if(data->state.engine) { ENGINE_finish(data->state.engine); ENGINE_free(data->state.engine); @@ -1922,7 +1969,15 @@ static void ssl_tls_trace(int direction, int ssl_ver, int content_type, } else #endif - { + if(content_type == SSL3_RT_CHANGE_CIPHER_SPEC) { + msg_type = *(char *)buf; + msg_name = "Change cipher spec"; + } + else if(content_type == SSL3_RT_ALERT) { + msg_type = (((char *)buf)[0] << 8) + ((char *)buf)[1]; + msg_name = SSL_alert_desc_string_long(msg_type); + } + else { msg_type = *(char *)buf; msg_name = ssl_msg_type(ssl_ver, msg_type); } @@ -1931,12 +1986,12 @@ static void ssl_tls_trace(int direction, int ssl_ver, int content_type, verstr, direction?"OUT":"IN", tls_rt_name, msg_name, msg_type); if(0 <= txt_len && (unsigned)txt_len < sizeof(ssl_buf)) { - Curl_debug(data, CURLINFO_TEXT, ssl_buf, (size_t)txt_len, NULL); + Curl_debug(data, CURLINFO_TEXT, ssl_buf, (size_t)txt_len); } } Curl_debug(data, (direction == 1) ? CURLINFO_SSL_DATA_OUT : - CURLINFO_SSL_DATA_IN, (char *)buf, len, NULL); + CURLINFO_SSL_DATA_IN, (char *)buf, len); (void) ssl; } #endif @@ -2064,10 +2119,6 @@ set_ssl_version_min_max(long *ctx_options, struct connectdata *conn, long ssl_version = SSL_CONN_CONFIG(version); long ssl_version_max = SSL_CONN_CONFIG(version_max); - if(ssl_version_max == CURL_SSLVERSION_MAX_NONE) { - ssl_version_max = ssl_version << 16; - } - switch(ssl_version) { case CURL_SSLVERSION_TLSv1_3: #ifdef TLS1_3_VERSION @@ -2099,8 +2150,7 @@ set_ssl_version_min_max(long *ctx_options, struct connectdata *conn, #endif /* FALLTHROUGH */ case CURL_SSLVERSION_TLSv1_0: - *ctx_options |= SSL_OP_NO_SSLv2; - *ctx_options |= SSL_OP_NO_SSLv3; + case CURL_SSLVERSION_TLSv1: break; } @@ -2116,12 +2166,12 @@ set_ssl_version_min_max(long *ctx_options, struct connectdata *conn, #endif /* FALLTHROUGH */ case CURL_SSLVERSION_MAX_TLSv1_2: - case CURL_SSLVERSION_MAX_DEFAULT: #ifdef TLS1_3_VERSION *ctx_options |= SSL_OP_NO_TLSv1_3; #endif break; case CURL_SSLVERSION_MAX_TLSv1_3: + case CURL_SSLVERSION_MAX_DEFAULT: #ifdef TLS1_3_VERSION break; #else @@ -2305,11 +2355,6 @@ static CURLcode ossl_connect_step1(struct connectdata *conn, int sockindex) switch(ssl_version) { case CURL_SSLVERSION_SSLv3: -#ifdef USE_TLS_SRP - if(ssl_authtype == CURL_TLSAUTH_SRP) { - infof(data, "Set version TLSv1.x for SRP authorisation\n"); - } -#endif ctx_options |= SSL_OP_NO_SSLv2; ctx_options |= SSL_OP_NO_TLSv1; #if OPENSSL_VERSION_NUMBER >= 0x1000100FL @@ -2323,21 +2368,20 @@ static CURLcode ossl_connect_step1(struct connectdata *conn, int sockindex) case CURL_SSLVERSION_DEFAULT: case CURL_SSLVERSION_TLSv1: - ctx_options |= SSL_OP_NO_SSLv2; - ctx_options |= SSL_OP_NO_SSLv3; - break; - case CURL_SSLVERSION_TLSv1_0: case CURL_SSLVERSION_TLSv1_1: case CURL_SSLVERSION_TLSv1_2: case CURL_SSLVERSION_TLSv1_3: + /* asking for any TLS version as the minimum, means no SSL versions + allowed */ + ctx_options |= SSL_OP_NO_SSLv2; + ctx_options |= SSL_OP_NO_SSLv3; result = set_ssl_version_min_max(&ctx_options, conn, sockindex); if(result != CURLE_OK) return result; break; case CURL_SSLVERSION_SSLv2: -#ifndef OPENSSL_NO_SSL2 ctx_options |= SSL_OP_NO_SSLv3; ctx_options |= SSL_OP_NO_TLSv1; #if OPENSSL_VERSION_NUMBER >= 0x1000100FL @@ -2348,10 +2392,6 @@ static CURLcode ossl_connect_step1(struct connectdata *conn, int sockindex) #endif #endif break; -#else - failf(data, OSSL_PACKAGE " was built without SSLv2 support"); - return CURLE_NOT_BUILT_IN; -#endif default: failf(data, "Unrecognized parameter passed via CURLOPT_SSLVERSION"); @@ -2414,6 +2454,19 @@ static CURLcode ossl_connect_step1(struct connectdata *conn, int sockindex) infof(data, "Cipher selection: %s\n", ciphers); } +#ifdef HAVE_SSL_CTX_SET_CIPHERSUITES + { + char *ciphers13 = SSL_CONN_CONFIG(cipher_list13); + if(ciphers13) { + if(!SSL_CTX_set_ciphersuites(BACKEND->ctx, ciphers13)) { + failf(data, "failed setting TLS 1.3 cipher suite: %s", ciphers); + return CURLE_SSL_CIPHER; + } + infof(data, "TLS 1.3 cipher selection: %s\n", ciphers13); + } + } +#endif + #ifdef USE_TLS_SRP if(ssl_authtype == CURL_TLSAUTH_SRP) { char * const ssl_username = SSL_SET_OPTION(username); @@ -3754,6 +3807,9 @@ const struct Curl_ssl Curl_ssl_openssl = { SSLSUPP_CERTINFO | SSLSUPP_PINNEDPUBKEY | SSLSUPP_SSL_CTX | +#ifdef HAVE_SSL_CTX_SET_CIPHERSUITES + SSLSUPP_TLS13_CIPHERSUITES | +#endif SSLSUPP_HTTPS_PROXY, sizeof(struct ssl_backend_data), diff --git a/Utilities/cmcurl/lib/vtls/schannel.c b/Utilities/cmcurl/lib/vtls/schannel.c index 2cfd5c1..8f6c301 100644 --- a/Utilities/cmcurl/lib/vtls/schannel.c +++ b/Utilities/cmcurl/lib/vtls/schannel.c @@ -90,11 +90,17 @@ #endif #endif +#if defined(CryptStringToBinary) && defined(CRYPT_STRING_HEX) +#define HAS_CLIENT_CERT_PATH +#endif + +#ifdef HAS_CLIENT_CERT_PATH #ifdef UNICODE #define CURL_CERT_STORE_PROV_SYSTEM CERT_STORE_PROV_SYSTEM_W #else #define CURL_CERT_STORE_PROV_SYSTEM CERT_STORE_PROV_SYSTEM_A #endif +#endif #ifndef SP_PROT_SSL2_CLIENT #define SP_PROT_SSL2_CLIENT 0x00000008 @@ -199,6 +205,155 @@ set_ssl_version_min_max(SCHANNEL_CRED *schannel_cred, struct connectdata *conn) return CURLE_OK; } +/*longest is 26, buffer is slightly bigger*/ +#define LONGEST_ALG_ID 32 +#define CIPHEROPTION(X) \ +if(strcmp(#X, tmp) == 0) \ + return X + +static int +get_alg_id_by_name(char *name) +{ + char tmp[LONGEST_ALG_ID] = { 0 }; + char *nameEnd = strchr(name, ':'); + size_t n = nameEnd ? min((size_t)(nameEnd - name), LONGEST_ALG_ID - 1) : \ + min(strlen(name), LONGEST_ALG_ID - 1); + strncpy(tmp, name, n); + tmp[n] = 0; + CIPHEROPTION(CALG_MD2); + CIPHEROPTION(CALG_MD4); + CIPHEROPTION(CALG_MD5); + CIPHEROPTION(CALG_SHA); + CIPHEROPTION(CALG_SHA1); + CIPHEROPTION(CALG_MAC); + CIPHEROPTION(CALG_RSA_SIGN); + CIPHEROPTION(CALG_DSS_SIGN); +/*ifdefs for the options that are defined conditionally in wincrypt.h*/ +#ifdef CALG_NO_SIGN + CIPHEROPTION(CALG_NO_SIGN); +#endif + CIPHEROPTION(CALG_RSA_KEYX); + CIPHEROPTION(CALG_DES); +#ifdef CALG_3DES_112 + CIPHEROPTION(CALG_3DES_112); +#endif + CIPHEROPTION(CALG_3DES); + CIPHEROPTION(CALG_DESX); + CIPHEROPTION(CALG_RC2); + CIPHEROPTION(CALG_RC4); + CIPHEROPTION(CALG_SEAL); +#ifdef CALG_DH_SF + CIPHEROPTION(CALG_DH_SF); +#endif + CIPHEROPTION(CALG_DH_EPHEM); +#ifdef CALG_AGREEDKEY_ANY + CIPHEROPTION(CALG_AGREEDKEY_ANY); +#endif +#ifdef CALG_HUGHES_MD5 + CIPHEROPTION(CALG_HUGHES_MD5); +#endif + CIPHEROPTION(CALG_SKIPJACK); +#ifdef CALG_TEK + CIPHEROPTION(CALG_TEK); +#endif + CIPHEROPTION(CALG_CYLINK_MEK); + CIPHEROPTION(CALG_SSL3_SHAMD5); +#ifdef CALG_SSL3_MASTER + CIPHEROPTION(CALG_SSL3_MASTER); +#endif +#ifdef CALG_SCHANNEL_MASTER_HASH + CIPHEROPTION(CALG_SCHANNEL_MASTER_HASH); +#endif +#ifdef CALG_SCHANNEL_MAC_KEY + CIPHEROPTION(CALG_SCHANNEL_MAC_KEY); +#endif +#ifdef CALG_SCHANNEL_ENC_KEY + CIPHEROPTION(CALG_SCHANNEL_ENC_KEY); +#endif +#ifdef CALG_PCT1_MASTER + CIPHEROPTION(CALG_PCT1_MASTER); +#endif +#ifdef CALG_SSL2_MASTER + CIPHEROPTION(CALG_SSL2_MASTER); +#endif +#ifdef CALG_TLS1_MASTER + CIPHEROPTION(CALG_TLS1_MASTER); +#endif +#ifdef CALG_RC5 + CIPHEROPTION(CALG_RC5); +#endif +#ifdef CALG_HMAC + CIPHEROPTION(CALG_HMAC); +#endif +#if !defined(__W32API_MAJOR_VERSION) || \ + !defined(__W32API_MINOR_VERSION) || \ + defined(__MINGW64_VERSION_MAJOR) || \ + (__W32API_MAJOR_VERSION > 5) || \ + ((__W32API_MAJOR_VERSION == 5) && (__W32API_MINOR_VERSION > 0)) + /* CALG_TLS1PRF has a syntax error in MinGW's w32api up to version 5.0, + see https://osdn.net/projects/mingw/ticket/38391 */ + CIPHEROPTION(CALG_TLS1PRF); +#endif +#ifdef CALG_HASH_REPLACE_OWF + CIPHEROPTION(CALG_HASH_REPLACE_OWF); +#endif +#ifdef CALG_AES_128 + CIPHEROPTION(CALG_AES_128); +#endif +#ifdef CALG_AES_192 + CIPHEROPTION(CALG_AES_192); +#endif +#ifdef CALG_AES_256 + CIPHEROPTION(CALG_AES_256); +#endif +#ifdef CALG_AES + CIPHEROPTION(CALG_AES); +#endif +#ifdef CALG_SHA_256 + CIPHEROPTION(CALG_SHA_256); +#endif +#ifdef CALG_SHA_384 + CIPHEROPTION(CALG_SHA_384); +#endif +#ifdef CALG_SHA_512 + CIPHEROPTION(CALG_SHA_512); +#endif +#ifdef CALG_ECDH + CIPHEROPTION(CALG_ECDH); +#endif +#ifdef CALG_ECMQV + CIPHEROPTION(CALG_ECMQV); +#endif +#ifdef CALG_ECDSA + CIPHEROPTION(CALG_ECDSA); +#endif + return 0; +} + +static CURLcode +set_ssl_ciphers(SCHANNEL_CRED *schannel_cred, char *ciphers) +{ + char *startCur = ciphers; + int algCount = 0; + static ALG_ID algIds[45]; /*There are 45 listed in the MS headers*/ + while(startCur && (0 != *startCur) && (algCount < 45)) { + long alg = strtol(startCur, 0, 0); + if(!alg) + alg = get_alg_id_by_name(startCur); + if(alg) + algIds[algCount++] = alg; + else + return CURLE_SSL_CIPHER; + startCur = strchr(startCur, ':'); + if(startCur) + startCur++; + } + schannel_cred->palgSupportedAlgs = algIds; + schannel_cred->cSupportedAlgs = algCount; + return CURLE_OK; +} + +#ifdef HAS_CLIENT_CERT_PATH static CURLcode get_cert_location(TCHAR *path, DWORD *store_name, TCHAR **store_path, TCHAR **thumbprint) @@ -248,6 +403,7 @@ get_cert_location(TCHAR *path, DWORD *store_name, TCHAR **store_path, return CURLE_OK; } +#endif static CURLcode schannel_connect_step1(struct connectdata *conn, int sockindex) @@ -299,10 +455,15 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) #endif #ifdef _WIN32_WCE +#ifdef HAS_MANUAL_VERIFY_API /* certificate validation on CE doesn't seem to work right; we'll * do it following a more manual process. */ BACKEND->use_manual_cred_validation = true; #else +#error "compiler too old to support requisite manual cert verify for Win CE" +#endif +#else +#ifdef HAS_MANUAL_VERIFY_API if(SSL_CONN_CONFIG(CAfile)) { if(Curl_verify_windows_version(6, 1, PLATFORM_WINNT, VERSION_GREATER_THAN_EQUAL)) { @@ -316,6 +477,12 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) } else BACKEND->use_manual_cred_validation = false; +#else + if(SSL_CONN_CONFIG(CAfile)) { + failf(data, "schannel: CA cert support not built in"); + return CURLE_NOT_BUILT_IN; + } +#endif #endif BACKEND->cred = NULL; @@ -341,9 +508,11 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) schannel_cred.dwVersion = SCHANNEL_CRED_VERSION; if(conn->ssl_config.verifypeer) { +#ifdef HAS_MANUAL_VERIFY_API if(BACKEND->use_manual_cred_validation) schannel_cred.dwFlags = SCH_CRED_MANUAL_CRED_VALIDATION; else +#endif schannel_cred.dwFlags = SCH_CRED_AUTO_CRED_VALIDATION; /* TODO s/data->set.ssl.no_revoke/SSL_SET_OPTION(no_revoke)/g */ @@ -401,6 +570,16 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) return CURLE_SSL_CONNECT_ERROR; } + if(SSL_CONN_CONFIG(cipher_list)) { + result = set_ssl_ciphers(&schannel_cred, SSL_CONN_CONFIG(cipher_list)); + if(CURLE_OK != result) { + failf(data, "Unable to set ciphers to passed via SSL_CONN_CONFIG"); + return result; + } + } + + +#ifdef HAS_CLIENT_CERT_PATH /* client certificate */ if(data->set.ssl.cert) { DWORD cert_store_name; @@ -417,14 +596,21 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) result = get_cert_location(cert_path, &cert_store_name, &cert_store_path, &cert_thumbprint_str); if(result != CURLE_OK) { + failf(data, "schannel: Failed to get certificate location for %s", + cert_path); Curl_unicodefree(cert_path); return result; } - cert_store = CertOpenStore(CURL_CERT_STORE_PROV_SYSTEM, 0, - (HCRYPTPROV)NULL, - cert_store_name, cert_store_path); + cert_store = + CertOpenStore(CURL_CERT_STORE_PROV_SYSTEM, 0, + (HCRYPTPROV)NULL, + CERT_STORE_OPEN_EXISTING_FLAG | cert_store_name, + cert_store_path); if(!cert_store) { + failf(data, "schannel: Failed to open cert store %x %s, " + "last error is %x", + cert_store_name, cert_store_path, GetLastError()); Curl_unicodefree(cert_path); return CURLE_SSL_CONNECT_ERROR; } @@ -453,6 +639,12 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) CertCloseStore(cert_store, 0); } +#else + if(data->set.ssl.cert) { + failf(data, "schannel: client cert support not built in"); + return CURLE_NOT_BUILT_IN; + } +#endif /* allocate memory for the re-usable credential handle */ BACKEND->cred = (struct curl_schannel_cred *) @@ -877,9 +1069,11 @@ schannel_connect_step2(struct connectdata *conn, int sockindex) } } +#ifdef HAS_MANUAL_VERIFY_API if(conn->ssl_config.verifypeer && BACKEND->use_manual_cred_validation) { return verify_certificate(conn, sockindex); } +#endif return CURLE_OK; } @@ -1819,7 +2013,7 @@ static CURLcode pkp_pin_peer_pubkey(struct connectdata *conn, int sockindex, x509_der = (const char *)pCertContextServer->pbCertEncoded; x509_der_len = pCertContextServer->cbCertEncoded; - memset(&x509_parsed, 0, sizeof x509_parsed); + memset(&x509_parsed, 0, sizeof(x509_parsed)); if(Curl_parseX509(&x509_parsed, x509_der, x509_der + x509_der_len)) break; @@ -1870,7 +2064,8 @@ static void Curl_schannel_checksum(const unsigned char *input, if(!CryptCreateHash(hProv, algId, 0, 0, &hHash)) break; /* failed */ - if(!CryptHashData(hHash, (const BYTE*)input, (DWORD)inputlen, 0)) + /* workaround for original MinGW, should be (const BYTE*) */ + if(!CryptHashData(hHash, (BYTE*)input, (DWORD)inputlen, 0)) break; /* failed */ /* get hash size */ diff --git a/Utilities/cmcurl/lib/vtls/schannel.h b/Utilities/cmcurl/lib/vtls/schannel.h index 4476900..51417af 100644 --- a/Utilities/cmcurl/lib/vtls/schannel.h +++ b/Utilities/cmcurl/lib/vtls/schannel.h @@ -32,12 +32,44 @@ #include "urldata.h" +/* <wincrypt.h> has been included via the above <schnlsp.h>. + * Or in case of ldap.c, it was included via <winldap.h>. + * And since <wincrypt.h> has this: + * #define X509_NAME ((LPCSTR) 7) + * + * And in BoringSSL's <openssl/base.h> there is: + * typedef struct X509_name_st X509_NAME; + * etc. + * + * this wil cause all kinds of C-preprocessing paste errors in + * BoringSSL's <openssl/x509.h>: So just undefine those defines here + * (and only here). + */ +#if defined(HAVE_BORINGSSL) || defined(OPENSSL_IS_BORINGSSL) +# undef X509_NAME +# undef X509_CERT_PAIR +# undef X509_EXTENSIONS +#endif + extern const struct Curl_ssl Curl_ssl_schannel; CURLcode verify_certificate(struct connectdata *conn, int sockindex); /* structs to expose only in schannel.c and schannel_verify.c */ #ifdef EXPOSE_SCHANNEL_INTERNAL_STRUCTS + +#ifdef __MINGW32__ +#include <_mingw.h> +#ifdef __MINGW64_VERSION_MAJOR +#define HAS_MANUAL_VERIFY_API +#endif +#else +#include <wincrypt.h> +#ifdef CERT_CHAIN_REVOCATION_CHECK_CHAIN +#define HAS_MANUAL_VERIFY_API +#endif +#endif + struct curl_schannel_cred { CredHandle cred_handle; TimeStamp time_stamp; @@ -66,7 +98,9 @@ struct ssl_backend_data { bool recv_sspi_close_notify; /* true if connection closed by close_notify */ bool recv_connection_closed; /* true if connection closed, regardless how */ bool use_alpn; /* true if ALPN is used for this connection */ +#ifdef HAS_MANUAL_VERIFY_API bool use_manual_cred_validation; /* true if manual cred validation is used */ +#endif }; #endif /* EXPOSE_SCHANNEL_INTERNAL_STRUCTS */ diff --git a/Utilities/cmcurl/lib/vtls/schannel_verify.c b/Utilities/cmcurl/lib/vtls/schannel_verify.c index db187dd..5a7092a 100644 --- a/Utilities/cmcurl/lib/vtls/schannel_verify.c +++ b/Utilities/cmcurl/lib/vtls/schannel_verify.c @@ -30,14 +30,15 @@ #include "curl_setup.h" #ifdef USE_SCHANNEL - -#define EXPOSE_SCHANNEL_INTERNAL_STRUCTS - #ifndef USE_WINDOWS_SSPI # error "Can't compile SCHANNEL support without SSPI." #endif +#define EXPOSE_SCHANNEL_INTERNAL_STRUCTS #include "schannel.h" + +#ifdef HAS_MANUAL_VERIFY_API + #include "vtls.h" #include "sendf.h" #include "strerror.h" @@ -53,7 +54,7 @@ #define BACKEND connssl->backend #define MAX_CAFILE_SIZE 1048576 /* 1 MiB */ -#define BEGIN_CERT "-----BEGIN CERTIFICATE-----\n" +#define BEGIN_CERT "-----BEGIN CERTIFICATE-----" #define END_CERT "\n-----END CERTIFICATE-----" typedef struct { @@ -71,6 +72,10 @@ typedef struct { HCERTSTORE hExclusiveTrustedPeople; } CERT_CHAIN_ENGINE_CONFIG_WIN7, *PCERT_CHAIN_ENGINE_CONFIG_WIN7; +static int is_cr_or_lf(char c) +{ + return c == '\r' || c == '\n'; +} static CURLcode add_certs_to_store(HCERTSTORE trust_store, const char *ca_file, @@ -177,7 +182,7 @@ static CURLcode add_certs_to_store(HCERTSTORE trust_store, current_ca_file_ptr = ca_file_buffer; while(more_certs && *current_ca_file_ptr != '\0') { char *begin_cert_ptr = strstr(current_ca_file_ptr, BEGIN_CERT); - if(!begin_cert_ptr) { + if(!begin_cert_ptr || !is_cr_or_lf(begin_cert_ptr[strlen(BEGIN_CERT)])) { more_certs = 0; } else { @@ -209,7 +214,7 @@ static CURLcode add_certs_to_store(HCERTSTORE trust_store, NULL, NULL, NULL, - &cert_context)) { + (const void **)&cert_context)) { failf(data, "schannel: failed to extract certificate from CA file " @@ -548,4 +553,5 @@ CURLcode verify_certificate(struct connectdata *conn, int sockindex) return result; } +#endif /* HAS_MANUAL_VERIFY_API */ #endif /* USE_SCHANNEL */ diff --git a/Utilities/cmcurl/lib/vtls/vtls.c b/Utilities/cmcurl/lib/vtls/vtls.c index ee5bc7a..b61c640 100644 --- a/Utilities/cmcurl/lib/vtls/vtls.c +++ b/Utilities/cmcurl/lib/vtls/vtls.c @@ -96,7 +96,8 @@ Curl_ssl_config_matches(struct ssl_primary_config* data, Curl_safe_strcasecompare(data->clientcert, needle->clientcert) && Curl_safe_strcasecompare(data->random_file, needle->random_file) && Curl_safe_strcasecompare(data->egdsocket, needle->egdsocket) && - Curl_safe_strcasecompare(data->cipher_list, needle->cipher_list)) + Curl_safe_strcasecompare(data->cipher_list, needle->cipher_list) && + Curl_safe_strcasecompare(data->cipher_list13, needle->cipher_list13)) return TRUE; return FALSE; @@ -119,6 +120,7 @@ Curl_clone_primary_ssl_config(struct ssl_primary_config *source, CLONE_STRING(random_file); CLONE_STRING(egdsocket); CLONE_STRING(cipher_list); + CLONE_STRING(cipher_list13); return TRUE; } @@ -131,6 +133,7 @@ void Curl_free_primary_ssl_config(struct ssl_primary_config* sslc) Curl_safefree(sslc->random_file); Curl_safefree(sslc->egdsocket); Curl_safefree(sslc->cipher_list); + Curl_safefree(sslc->cipher_list13); } #ifdef USE_SSL @@ -980,6 +983,14 @@ bool Curl_ssl_false_start(void) } /* + * Check whether the SSL backend supports setting TLS 1.3 cipher suites + */ +bool Curl_ssl_tls13_ciphersuites(void) +{ + return Curl_ssl->supports & SSLSUPP_TLS13_CIPHERSUITES; +} + +/* * Default implementations for unsupported functions. */ diff --git a/Utilities/cmcurl/lib/vtls/vtls.h b/Utilities/cmcurl/lib/vtls/vtls.h index e7b87c4..40f9d74 100644 --- a/Utilities/cmcurl/lib/vtls/vtls.h +++ b/Utilities/cmcurl/lib/vtls/vtls.h @@ -31,6 +31,7 @@ struct ssl_connect_data; #define SSLSUPP_PINNEDPUBKEY (1<<2) /* supports CURLOPT_PINNEDPUBLICKEY */ #define SSLSUPP_SSL_CTX (1<<3) /* supports CURLOPT_SSL_CTX */ #define SSLSUPP_HTTPS_PROXY (1<<4) /* supports access via HTTPS proxies */ +#define SSLSUPP_TLS13_CIPHERSUITES (1<<5) /* supports TLS 1.3 ciphersuites */ struct Curl_ssl { /* @@ -93,6 +94,7 @@ CURLcode Curl_none_set_engine(struct Curl_easy *data, const char *engine); CURLcode Curl_none_set_engine_default(struct Curl_easy *data); struct curl_slist *Curl_none_engines_list(struct Curl_easy *data); bool Curl_none_false_start(void); +bool Curl_ssl_tls13_ciphersuites(void); CURLcode Curl_none_md5sum(unsigned char *input, size_t inputlen, unsigned char *md5sum, size_t md5len); @@ -246,7 +248,7 @@ bool Curl_ssl_false_start(void); #define SSL_SHUTDOWN_TIMEOUT 10000 /* ms */ -#else +#else /* if not USE_SSL */ /* When SSL support is not present, just define away these function calls */ #define Curl_ssl_init() 1 @@ -270,6 +272,7 @@ bool Curl_ssl_false_start(void); #define Curl_ssl_random(x,y,z) ((void)x, CURLE_NOT_BUILT_IN) #define Curl_ssl_cert_status_request() FALSE #define Curl_ssl_false_start() FALSE +#define Curl_ssl_tls13_ciphersuites() FALSE #endif #endif /* HEADER_CURL_VTLS_H */ diff --git a/Utilities/cmcurl/lib/warnless.h b/Utilities/cmcurl/lib/warnless.h index f6a2d74..284ea1e 100644 --- a/Utilities/cmcurl/lib/warnless.h +++ b/Utilities/cmcurl/lib/warnless.h @@ -26,6 +26,9 @@ #include <curl/curl.h> /* for curl_socket_t */ #endif +#define CURLX_FUNCTION_CAST(target_type, func) \ + (target_type)(void (*) (void))(func) + unsigned short curlx_ultous(unsigned long ulnum); unsigned char curlx_ultouc(unsigned long ulnum); diff --git a/Utilities/cmcurl/lib/x509asn1.c b/Utilities/cmcurl/lib/x509asn1.c index bba2023..72a0b4a 100644 --- a/Utilities/cmcurl/lib/x509asn1.c +++ b/Utilities/cmcurl/lib/x509asn1.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -40,10 +40,6 @@ #include "curl_memory.h" #include "memdebug.h" -/* For overflow checks. */ -#define CURL_SIZE_T_MAX ((size_t)-1) - - /* ASN.1 OIDs. */ static const char cnOID[] = "2.5.4.3"; /* Common name. */ static const char sanOID[] = "2.5.29.17"; /* Subject alternative name. */ @@ -108,8 +104,8 @@ static const curl_OID OIDtable[] = { */ -const char *Curl_getASN1Element(curl_asn1Element *elem, - const char *beg, const char *end) +static const char *getASN1Element(curl_asn1Element *elem, + const char *beg, const char *end) { unsigned char b; unsigned long len; @@ -146,7 +142,7 @@ const char *Curl_getASN1Element(curl_asn1Element *elem, return (const char *) NULL; elem->beg = beg; while(beg < end && *beg) { - beg = Curl_getASN1Element(&lelem, beg, end); + beg = getASN1Element(&lelem, beg, end); if(!beg) return (const char *) NULL; } @@ -206,7 +202,7 @@ static const char *octet2str(const char *beg, const char *end) /* Convert an ASN.1 octet string to a printable string. Return the dynamically allocated string, or NULL if an error occurs. */ - if(n <= (CURL_SIZE_T_MAX - 1) / 3) { + if(n <= (SIZE_T_MAX - 1) / 3) { buf = malloc(3 * n + 1); if(buf) for(n = 0; beg < end; n += 3) @@ -287,7 +283,7 @@ utf8asn1str(char **to, int type, const char *from, const char *end) if(inlength % size) return -1; /* Length inconsistent with character size. */ - if(inlength / size > (CURL_SIZE_T_MAX - 1) / 4) + if(inlength / size > (SIZE_T_MAX - 1) / 4) return -1; /* Too big. */ buf = malloc(4 * (inlength / size) + 1); if(!buf) @@ -306,10 +302,10 @@ utf8asn1str(char **to, int type, const char *from, const char *end) case 4: wc = (wc << 8) | *(const unsigned char *) from++; wc = (wc << 8) | *(const unsigned char *) from++; - /* fallthrough */ + /* FALLTHROUGH */ case 2: wc = (wc << 8) | *(const unsigned char *) from++; - /* fallthrough */ + /* FALLTHROUGH */ default: /* case 1: */ wc = (wc << 8) | *(const unsigned char *) from++; } @@ -546,7 +542,7 @@ static const char *UTime2str(const char *beg, const char *end) tzl, tzp); } -const char *Curl_ASN1tostr(curl_asn1Element *elem, int type) +static const char *ASN1tostr(curl_asn1Element *elem, int type) { /* Convert an ASN.1 element to a printable string. Return the dynamically allocated string, or NULL if an error occurs. */ @@ -605,12 +601,12 @@ static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn) Return the total string length, even if larger than `n'. */ for(p1 = dn->beg; p1 < dn->end;) { - p1 = Curl_getASN1Element(&rdn, p1, dn->end); + p1 = getASN1Element(&rdn, p1, dn->end); for(p2 = rdn.beg; p2 < rdn.end;) { - p2 = Curl_getASN1Element(&atv, p2, rdn.end); - p3 = Curl_getASN1Element(&oid, atv.beg, atv.end); - Curl_getASN1Element(&value, p3, atv.end); - str = Curl_ASN1tostr(&oid, 0); + p2 = getASN1Element(&atv, p2, rdn.end); + p3 = getASN1Element(&oid, atv.beg, atv.end); + getASN1Element(&value, p3, atv.end); + str = ASN1tostr(&oid, 0); if(!str) return -1; @@ -640,7 +636,7 @@ static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn) l++; /* Generate value. */ - str = Curl_ASN1tostr(&value, 0); + str = ASN1tostr(&value, 0); if(!str) return -1; for(p3 = str; *p3; p3++) { @@ -655,7 +651,7 @@ static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn) return l; } -const char *Curl_DNtostr(curl_asn1Element *dn) +static const char *DNtostr(curl_asn1Element *dn) { char *buf = (char *) NULL; ssize_t n = encodeDN(buf, 0, dn); @@ -694,17 +690,17 @@ int Curl_parseX509(curl_X509certificate *cert, cert->certificate.end = end; /* Get the sequence content. */ - if(!Curl_getASN1Element(&elem, beg, end)) + if(!getASN1Element(&elem, beg, end)) return -1; /* Invalid bounds/size. */ beg = elem.beg; end = elem.end; /* Get tbsCertificate. */ - beg = Curl_getASN1Element(&tbsCertificate, beg, end); + beg = getASN1Element(&tbsCertificate, beg, end); /* Skip the signatureAlgorithm. */ - beg = Curl_getASN1Element(&cert->signatureAlgorithm, beg, end); + beg = getASN1Element(&cert->signatureAlgorithm, beg, end); /* Get the signatureValue. */ - Curl_getASN1Element(&cert->signature, beg, end); + getASN1Element(&cert->signature, beg, end); /* Parse TBSCertificate. */ beg = tbsCertificate.beg; @@ -712,29 +708,29 @@ int Curl_parseX509(curl_X509certificate *cert, /* Get optional version, get serialNumber. */ cert->version.header = NULL; cert->version.beg = &defaultVersion; - cert->version.end = &defaultVersion + sizeof defaultVersion;; - beg = Curl_getASN1Element(&elem, beg, end); + cert->version.end = &defaultVersion + sizeof(defaultVersion); + beg = getASN1Element(&elem, beg, end); if(elem.tag == 0) { - Curl_getASN1Element(&cert->version, elem.beg, elem.end); - beg = Curl_getASN1Element(&elem, beg, end); + getASN1Element(&cert->version, elem.beg, elem.end); + beg = getASN1Element(&elem, beg, end); } cert->serialNumber = elem; /* Get signature algorithm. */ - beg = Curl_getASN1Element(&cert->signatureAlgorithm, beg, end); + beg = getASN1Element(&cert->signatureAlgorithm, beg, end); /* Get issuer. */ - beg = Curl_getASN1Element(&cert->issuer, beg, end); + beg = getASN1Element(&cert->issuer, beg, end); /* Get notBefore and notAfter. */ - beg = Curl_getASN1Element(&elem, beg, end); - ccp = Curl_getASN1Element(&cert->notBefore, elem.beg, elem.end); - Curl_getASN1Element(&cert->notAfter, ccp, elem.end); + beg = getASN1Element(&elem, beg, end); + ccp = getASN1Element(&cert->notBefore, elem.beg, elem.end); + getASN1Element(&cert->notAfter, ccp, elem.end); /* Get subject. */ - beg = Curl_getASN1Element(&cert->subject, beg, end); + beg = getASN1Element(&cert->subject, beg, end); /* Get subjectPublicKeyAlgorithm and subjectPublicKey. */ - beg = Curl_getASN1Element(&cert->subjectPublicKeyInfo, beg, end); - ccp = Curl_getASN1Element(&cert->subjectPublicKeyAlgorithm, + beg = getASN1Element(&cert->subjectPublicKeyInfo, beg, end); + ccp = getASN1Element(&cert->subjectPublicKeyAlgorithm, cert->subjectPublicKeyInfo.beg, cert->subjectPublicKeyInfo.end); - Curl_getASN1Element(&cert->subjectPublicKey, ccp, + getASN1Element(&cert->subjectPublicKey, ccp, cert->subjectPublicKeyInfo.end); /* Get optional issuerUiqueID, subjectUniqueID and extensions. */ cert->issuerUniqueID.tag = cert->subjectUniqueID.tag = 0; @@ -745,19 +741,19 @@ int Curl_parseX509(curl_X509certificate *cert, cert->extensions.header = NULL; cert->extensions.beg = cert->extensions.end = ""; if(beg < end) - beg = Curl_getASN1Element(&elem, beg, end); + beg = getASN1Element(&elem, beg, end); if(elem.tag == 1) { cert->issuerUniqueID = elem; if(beg < end) - beg = Curl_getASN1Element(&elem, beg, end); + beg = getASN1Element(&elem, beg, end); } if(elem.tag == 2) { cert->subjectUniqueID = elem; if(beg < end) - beg = Curl_getASN1Element(&elem, beg, end); + beg = getASN1Element(&elem, beg, end); } if(elem.tag == 3) - Curl_getASN1Element(&cert->extensions, elem.beg, elem.end); + getASN1Element(&cert->extensions, elem.beg, elem.end); return 0; } @@ -785,12 +781,12 @@ static const char *dumpAlgo(curl_asn1Element *param, /* Get algorithm parameters and return algorithm name. */ - beg = Curl_getASN1Element(&oid, beg, end); + beg = getASN1Element(&oid, beg, end); param->header = NULL; param->tag = 0; param->beg = param->end = end; if(beg < end) - Curl_getASN1Element(param, beg, end); + getASN1Element(param, beg, end); return OID2str(oid.beg, oid.end, TRUE); } @@ -801,7 +797,7 @@ static void do_pubkey_field(struct Curl_easy *data, int certnum, /* Generate a certificate information record for the public key. */ - output = Curl_ASN1tostr(elem, 0); + output = ASN1tostr(elem, 0); if(output) { if(data->set.ssl.certinfo) Curl_ssl_push_certinfo(data, certnum, label, output); @@ -825,10 +821,10 @@ static void do_pubkey(struct Curl_easy *data, int certnum, /* Generate all information records for the public key. */ /* Get the public key (single element). */ - Curl_getASN1Element(&pk, pubkey->beg + 1, pubkey->end); + getASN1Element(&pk, pubkey->beg + 1, pubkey->end); if(strcasecompare(algo, "rsaEncryption")) { - p = Curl_getASN1Element(&elem, pk.beg, pk.end); + p = getASN1Element(&elem, pk.beg, pk.end); /* Compute key length. */ for(q = elem.beg; !*q && q < elem.end; q++) ; @@ -849,22 +845,22 @@ static void do_pubkey(struct Curl_easy *data, int certnum, } /* Generate coefficients. */ do_pubkey_field(data, certnum, "rsa(n)", &elem); - Curl_getASN1Element(&elem, p, pk.end); + getASN1Element(&elem, p, pk.end); do_pubkey_field(data, certnum, "rsa(e)", &elem); } else if(strcasecompare(algo, "dsa")) { - p = Curl_getASN1Element(&elem, param->beg, param->end); + p = getASN1Element(&elem, param->beg, param->end); do_pubkey_field(data, certnum, "dsa(p)", &elem); - p = Curl_getASN1Element(&elem, p, param->end); + p = getASN1Element(&elem, p, param->end); do_pubkey_field(data, certnum, "dsa(q)", &elem); - Curl_getASN1Element(&elem, p, param->end); + getASN1Element(&elem, p, param->end); do_pubkey_field(data, certnum, "dsa(g)", &elem); do_pubkey_field(data, certnum, "dsa(pub_key)", &pk); } else if(strcasecompare(algo, "dhpublicnumber")) { - p = Curl_getASN1Element(&elem, param->beg, param->end); + p = getASN1Element(&elem, param->beg, param->end); do_pubkey_field(data, certnum, "dh(p)", &elem); - Curl_getASN1Element(&elem, param->beg, param->end); + getASN1Element(&elem, param->beg, param->end); do_pubkey_field(data, certnum, "dh(g)", &elem); do_pubkey_field(data, certnum, "dh(pub_key)", &pk); } @@ -903,7 +899,7 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, return CURLE_OUT_OF_MEMORY; /* Subject. */ - ccp = Curl_DNtostr(&cert.subject); + ccp = DNtostr(&cert.subject); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -913,7 +909,7 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, free((char *) ccp); /* Issuer. */ - ccp = Curl_DNtostr(&cert.issuer); + ccp = DNtostr(&cert.issuer); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -937,7 +933,7 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, infof(data, " Version: %lu (0x%lx)\n", version + 1, version); /* Serial number. */ - ccp = Curl_ASN1tostr(&cert.serialNumber, 0); + ccp = ASN1tostr(&cert.serialNumber, 0); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -958,7 +954,7 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, free((char *) ccp); /* Start Date. */ - ccp = Curl_ASN1tostr(&cert.notBefore, 0); + ccp = ASN1tostr(&cert.notBefore, 0); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -968,7 +964,7 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, free((char *) ccp); /* Expire Date. */ - ccp = Curl_ASN1tostr(&cert.notAfter, 0); + ccp = ASN1tostr(&cert.notAfter, 0); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -992,7 +988,7 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, /* TODO: extensions. */ /* Signature. */ - ccp = Curl_ASN1tostr(&cert.signature, 0); + ccp = ASN1tostr(&cert.signature, 0); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -1051,7 +1047,7 @@ static const char *checkOID(const char *beg, const char *end, /* Check if first ASN.1 element at `beg' is the given OID. Return a pointer in the source after the OID if found, else NULL. */ - ccp = Curl_getASN1Element(&e, beg, end); + ccp = getASN1Element(&e, beg, end); if(!ccp || e.tag != CURL_ASN1_OBJECT_IDENTIFIER) return (const char *) NULL; @@ -1110,19 +1106,19 @@ CURLcode Curl_verifyhost(struct connectdata *conn, /* Process extensions. */ for(p = cert.extensions.beg; p < cert.extensions.end && matched != 1;) { - p = Curl_getASN1Element(&ext, p, cert.extensions.end); + p = getASN1Element(&ext, p, cert.extensions.end); /* Check if extension is a subjectAlternativeName. */ ext.beg = checkOID(ext.beg, ext.end, sanOID); if(ext.beg) { - ext.beg = Curl_getASN1Element(&elem, ext.beg, ext.end); + ext.beg = getASN1Element(&elem, ext.beg, ext.end); /* Skip critical if present. */ if(elem.tag == CURL_ASN1_BOOLEAN) - ext.beg = Curl_getASN1Element(&elem, ext.beg, ext.end); + ext.beg = getASN1Element(&elem, ext.beg, ext.end); /* Parse the octet string contents: is a single sequence. */ - Curl_getASN1Element(&elem, elem.beg, elem.end); + getASN1Element(&elem, elem.beg, elem.end); /* Check all GeneralNames. */ for(q = elem.beg; matched != 1 && q < elem.end;) { - q = Curl_getASN1Element(&name, q, elem.end); + q = getASN1Element(&name, q, elem.end); switch(name.tag) { case 2: /* DNS name. */ len = utf8asn1str(&dnsname, CURL_ASN1_IA5_STRING, @@ -1162,9 +1158,9 @@ CURLcode Curl_verifyhost(struct connectdata *conn, /* we have to look to the last occurrence of a commonName in the distinguished one to get the most significant one. */ while(q < cert.subject.end) { - q = Curl_getASN1Element(&dn, q, cert.subject.end); + q = getASN1Element(&dn, q, cert.subject.end); for(p = dn.beg; p < dn.end;) { - p = Curl_getASN1Element(&elem, p, dn.end); + p = getASN1Element(&elem, p, dn.end); /* We have a DN's AttributeTypeAndValue: check it in case it's a CN. */ elem.beg = checkOID(elem.beg, elem.end, cnOID); if(elem.beg) @@ -1173,7 +1169,7 @@ CURLcode Curl_verifyhost(struct connectdata *conn, } /* Check the CN if found. */ - if(!Curl_getASN1Element(&elem, name.beg, name.end)) + if(!getASN1Element(&elem, name.beg, name.end)) failf(data, "SSL: unable to obtain common name from peer certificate"); else { len = utf8asn1str(&dnsname, elem.tag, elem.beg, elem.end); diff --git a/Utilities/cmlibarchive/CMakeLists.txt b/Utilities/cmlibarchive/CMakeLists.txt index 206f3c6..d7af6e2 100644 --- a/Utilities/cmlibarchive/CMakeLists.txt +++ b/Utilities/cmlibarchive/CMakeLists.txt @@ -71,6 +71,7 @@ include(CTest) OPTION(ENABLE_NETTLE "Enable use of Nettle" ON) OPTION(ENABLE_OPENSSL "Enable use of OpenSSL" ON) +OPTION(ENABLE_LZ4 "Enable the use of the system LZ4 library if found" ON) OPTION(ENABLE_LZO "Enable the use of the system LZO library if found" OFF) OPTION(ENABLE_LZMA "Enable the use of the system LZMA library if found" ON) @@ -328,7 +329,7 @@ IF(ENABLE_LZO) INCLUDE(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZO2 DEFAULT_MSG LZO2_LIBRARY LZO2_INCLUDE_DIR) ELSE(ENABLE_LZO) - SET(LIBZMA_FOUND FALSE) # Override cached value + SET(LZO2_FOUND FALSE) # Override cached value ENDIF(ENABLE_LZO) IF(LZO2_FOUND) SET(HAVE_LIBLZO2 1) @@ -347,15 +348,19 @@ IF(0) # CMake does not need LZ4 support in libarchive # # Find LZ4 # -IF (LZ4_INCLUDE_DIR) - # Already in cache, be silent - SET(LZ4_FIND_QUIETLY TRUE) -ENDIF (LZ4_INCLUDE_DIR) +IF(ENABLE_LZ4) + IF (LZ4_INCLUDE_DIR) + # Already in cache, be silent + SET(LZ4_FIND_QUIETLY TRUE) + ENDIF (LZ4_INCLUDE_DIR) -FIND_PATH(LZ4_INCLUDE_DIR lz4.h) -FIND_LIBRARY(LZ4_LIBRARY NAMES lz4 liblz4) -INCLUDE(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZ4 DEFAULT_MSG LZ4_LIBRARY LZ4_INCLUDE_DIR) + FIND_PATH(LZ4_INCLUDE_DIR lz4.h) + FIND_LIBRARY(LZ4_LIBRARY NAMES lz4 liblz4) + INCLUDE(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZ4 DEFAULT_MSG LZ4_LIBRARY LZ4_INCLUDE_DIR) +ELSE(ENABLE_LZ4) + SET(LZ4_FOUND FALSE) # Override cached value +ENDIF(ENABLE_LZ4) IF(LZ4_FOUND) SET(HAVE_LIBLZ4 1) SET(HAVE_LZ4_H 1) @@ -371,6 +376,31 @@ IF(LZ4_FOUND) ENDIF(LZ4_FOUND) MARK_AS_ADVANCED(CLEAR LZ4_INCLUDE_DIR) MARK_AS_ADVANCED(CLEAR LZ4_LIBRARY) +# +# Find Zstd +# +IF (ZSTD_INCLUDE_DIR) + # Already in cache, be silent + SET(ZSTD_FIND_QUIETLY TRUE) +ENDIF (ZSTD_INCLUDE_DIR) + +FIND_PATH(ZSTD_INCLUDE_DIR zstd.h) +FIND_LIBRARY(ZSTD_LIBRARY NAMES zstd libzstd) +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(ZSTD DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR) +IF(ZSTD_FOUND) + SET(HAVE_ZSTD_H 1) + INCLUDE_DIRECTORIES(${ZSTD_INCLUDE_DIR}) + LIST(APPEND ADDITIONAL_LIBS ${ZSTD_LIBRARY}) + SET(CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY}) + SET(CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR}) + CHECK_FUNCTION_EXISTS(ZSTD_compressStream HAVE_LIBZSTD) + # + # TODO: test for static library. + # +ENDIF(ZSTD_FOUND) +MARK_AS_ADVANCED(CLEAR ZSTD_INCLUDE_DIR) +MARK_AS_ADVANCED(CLEAR ZSTD_LIBRARY) ENDIF() # @@ -451,6 +481,7 @@ LA_CHECK_INCLUDE_FILE("sys/select.h" HAVE_SYS_SELECT_H) LA_CHECK_INCLUDE_FILE("sys/stat.h" HAVE_SYS_STAT_H) LA_CHECK_INCLUDE_FILE("sys/statfs.h" HAVE_SYS_STATFS_H) LA_CHECK_INCLUDE_FILE("sys/statvfs.h" HAVE_SYS_STATVFS_H) +LA_CHECK_INCLUDE_FILE("sys/sysmacros.h" HAVE_SYS_SYSMACROS_H) LA_CHECK_INCLUDE_FILE("sys/time.h" HAVE_SYS_TIME_H) LA_CHECK_INCLUDE_FILE("sys/utime.h" HAVE_SYS_UTIME_H) LA_CHECK_INCLUDE_FILE("sys/utsname.h" HAVE_SYS_UTSNAME_H) diff --git a/Utilities/cmlibarchive/build/cmake/config.h.in b/Utilities/cmlibarchive/build/cmake/config.h.in index 368a451..1851d81 100644 --- a/Utilities/cmlibarchive/build/cmake/config.h.in +++ b/Utilities/cmlibarchive/build/cmake/config.h.in @@ -728,6 +728,9 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `z' library (-lz). */ #cmakedefine HAVE_LIBZ 1 +/* Define to 1 if you have the `zstd' library (-lzstd). */ +#cmakedefine HAVE_LIBZSTD 1 + /* Define to 1 if you have the <limits.h> header file. */ #cmakedefine HAVE_LIMITS_H 1 @@ -1071,6 +1074,10 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the <sys/stat.h> header file. */ #cmakedefine HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the <sys/sysmacros.h> header file. */ +#cmakedefine HAVE_SYS_SYSMACROS_H 1 + /* Define to 1 if you have the <sys/time.h> header file. */ #cmakedefine HAVE_SYS_TIME_H 1 @@ -1185,6 +1192,9 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the <zlib.h> header file. */ #cmakedefine HAVE_ZLIB_H 1 +/* Define to 1 if you have the <zstd.h> header file. */ +#cmakedefine HAVE_ZSTD_H 1 + /* Define to 1 if you have the `_ctime64_s' function. */ #cmakedefine HAVE__CTIME64_S 1 diff --git a/Utilities/cmlibarchive/build/pkgconfig/libarchive.pc.in b/Utilities/cmlibarchive/build/pkgconfig/libarchive.pc.in index 95d7159..4b631e6 100644 --- a/Utilities/cmlibarchive/build/pkgconfig/libarchive.pc.in +++ b/Utilities/cmlibarchive/build/pkgconfig/libarchive.pc.in @@ -7,5 +7,6 @@ Name: libarchive Description: library that can create and read several streaming archive formats Version: @VERSION@ Cflags: -I${includedir} +Cflags.private: -DLIBARCHIVE_STATIC Libs: -L${libdir} -larchive Libs.private: @LIBS@ diff --git a/Utilities/cmlibarchive/build/version b/Utilities/cmlibarchive/build/version index 2dd0839..2427eab 100644 --- a/Utilities/cmlibarchive/build/version +++ b/Utilities/cmlibarchive/build/version @@ -1 +1 @@ -3003002 +3003003 diff --git a/Utilities/cmlibarchive/libarchive/CMakeLists.txt b/Utilities/cmlibarchive/libarchive/CMakeLists.txt index b02ae82..e38d664 100644 --- a/Utilities/cmlibarchive/libarchive/CMakeLists.txt +++ b/Utilities/cmlibarchive/libarchive/CMakeLists.txt @@ -88,6 +88,7 @@ SET(libarchive_SOURCES archive_read_support_filter_rpm.c archive_read_support_filter_uu.c archive_read_support_filter_xz.c + archive_read_support_filter_zstd.c archive_read_support_format_7zip.c archive_read_support_format_all.c archive_read_support_format_ar.c @@ -134,6 +135,7 @@ SET(libarchive_SOURCES archive_write_add_filter_program.c archive_write_add_filter_uuencode.c archive_write_add_filter_xz.c + archive_write_add_filter_zstd.c archive_write_set_format.c archive_write_set_format_7zip.c archive_write_set_format_ar.c diff --git a/Utilities/cmlibarchive/libarchive/archive.h b/Utilities/cmlibarchive/libarchive/archive.h index 0f94d2e..f3ebbfe 100644 --- a/Utilities/cmlibarchive/libarchive/archive.h +++ b/Utilities/cmlibarchive/libarchive/archive.h @@ -36,7 +36,7 @@ * assert that ARCHIVE_VERSION_NUMBER >= 2012108. */ /* Note: Compiler will complain if this does not match archive_entry.h! */ -#define ARCHIVE_VERSION_NUMBER 3003002 +#define ARCHIVE_VERSION_NUMBER 3003003 #include <sys/stat.h> #include <stddef.h> /* for wchar_t */ @@ -152,7 +152,7 @@ __LA_DECL int archive_version_number(void); /* * Textual name/version of the library, useful for version displays. */ -#define ARCHIVE_VERSION_ONLY_STRING "3.3.2" +#define ARCHIVE_VERSION_ONLY_STRING "3.3.3" #define ARCHIVE_VERSION_STRING "libarchive " ARCHIVE_VERSION_ONLY_STRING __LA_DECL const char * archive_version_string(void); @@ -174,6 +174,7 @@ __LA_DECL const char * archive_zlib_version(void); __LA_DECL const char * archive_liblzma_version(void); __LA_DECL const char * archive_bzlib_version(void); __LA_DECL const char * archive_liblz4_version(void); +__LA_DECL const char * archive_libzstd_version(void); /* Declare our basic types. */ struct archive; @@ -273,6 +274,7 @@ typedef const char *archive_passphrase_callback(struct archive *, #define ARCHIVE_FILTER_LZOP 11 #define ARCHIVE_FILTER_GRZIP 12 #define ARCHIVE_FILTER_LZ4 13 +#define ARCHIVE_FILTER_ZSTD 14 #if ARCHIVE_VERSION_NUMBER < 4000000 #define ARCHIVE_COMPRESSION_NONE ARCHIVE_FILTER_NONE @@ -430,6 +432,7 @@ __LA_DECL int archive_read_support_filter_program_signature __LA_DECL int archive_read_support_filter_rpm(struct archive *); __LA_DECL int archive_read_support_filter_uu(struct archive *); __LA_DECL int archive_read_support_filter_xz(struct archive *); +__LA_DECL int archive_read_support_filter_zstd(struct archive *); __LA_DECL int archive_read_support_format_7zip(struct archive *); __LA_DECL int archive_read_support_format_all(struct archive *); @@ -775,6 +778,7 @@ __LA_DECL int archive_write_add_filter_program(struct archive *, const char *cmd); __LA_DECL int archive_write_add_filter_uuencode(struct archive *); __LA_DECL int archive_write_add_filter_xz(struct archive *); +__LA_DECL int archive_write_add_filter_zstd(struct archive *); /* A convenience function to set the format based on the code or name. */ diff --git a/Utilities/cmlibarchive/libarchive/archive_acl.c b/Utilities/cmlibarchive/libarchive/archive_acl.c index b8b6b63..4736531 100644 --- a/Utilities/cmlibarchive/libarchive/archive_acl.c +++ b/Utilities/cmlibarchive/libarchive/archive_acl.c @@ -1159,6 +1159,7 @@ archive_acl_from_text_w(struct archive_acl *acl, const wchar_t *text, switch (want_type) { case ARCHIVE_ENTRY_ACL_TYPE_POSIX1E: want_type = ARCHIVE_ENTRY_ACL_TYPE_ACCESS; + __LA_FALLTHROUGH; case ARCHIVE_ENTRY_ACL_TYPE_ACCESS: case ARCHIVE_ENTRY_ACL_TYPE_DEFAULT: numfields = 5; @@ -1626,6 +1627,7 @@ archive_acl_from_text_l(struct archive_acl *acl, const char *text, switch (want_type) { case ARCHIVE_ENTRY_ACL_TYPE_POSIX1E: want_type = ARCHIVE_ENTRY_ACL_TYPE_ACCESS; + __LA_FALLTHROUGH; case ARCHIVE_ENTRY_ACL_TYPE_ACCESS: case ARCHIVE_ENTRY_ACL_TYPE_DEFAULT: numfields = 5; diff --git a/Utilities/cmlibarchive/libarchive/archive_cmdline.c b/Utilities/cmlibarchive/libarchive/archive_cmdline.c index 7d3bac5..5c519cd 100644 --- a/Utilities/cmlibarchive/libarchive/archive_cmdline.c +++ b/Utilities/cmlibarchive/libarchive/archive_cmdline.c @@ -100,10 +100,10 @@ get_argument(struct archive_string *as, const char *p) /* * Set up command line arguments. - * Returns ARChIVE_OK if everything okey. - * Returns ARChIVE_FAILED if there is a lack of the `"' terminator or an + * Returns ARCHIVE_OK if everything okey. + * Returns ARCHIVE_FAILED if there is a lack of the `"' terminator or an * empty command line. - * Returns ARChIVE_FATAL if no memory. + * Returns ARCHIVE_FATAL if no memory. */ int __archive_cmdline_parse(struct archive_cmdline *data, const char *cmd) diff --git a/Utilities/cmlibarchive/libarchive/archive_cryptor.c b/Utilities/cmlibarchive/libarchive/archive_cryptor.c index ced52fd..71967c9 100644 --- a/Utilities/cmlibarchive/libarchive/archive_cryptor.c +++ b/Utilities/cmlibarchive/libarchive/archive_cryptor.c @@ -153,7 +153,7 @@ aes_ctr_encrypt_counter(archive_crypto_ctx *ctx) CCCryptorStatus r; r = CCCryptorReset(ref, NULL); - if (r != kCCSuccess) + if (r != kCCSuccess && r != kCCUnimplemented) return -1; r = CCCryptorUpdate(ref, ctx->nonce, AES_BLOCK_SIZE, ctx->encr_buf, AES_BLOCK_SIZE, NULL); diff --git a/Utilities/cmlibarchive/libarchive/archive_cryptor_private.h b/Utilities/cmlibarchive/libarchive/archive_cryptor_private.h index 0ca544b..b975922 100644 --- a/Utilities/cmlibarchive/libarchive/archive_cryptor_private.h +++ b/Utilities/cmlibarchive/libarchive/archive_cryptor_private.h @@ -64,7 +64,7 @@ typedef struct { } archive_crypto_ctx; #elif defined(_WIN32) && !defined(__CYGWIN__) && defined(HAVE_BCRYPT_H) -#include <Bcrypt.h> +#include <bcrypt.h> /* Common in other bcrypt implementations, but missing from VS2008. */ #ifndef BCRYPT_SUCCESS diff --git a/Utilities/cmlibarchive/libarchive/archive_disk_acl_freebsd.c b/Utilities/cmlibarchive/libarchive/archive_disk_acl_freebsd.c index 07d08ff..aba41e5 100644 --- a/Utilities/cmlibarchive/libarchive/archive_disk_acl_freebsd.c +++ b/Utilities/cmlibarchive/libarchive/archive_disk_acl_freebsd.c @@ -93,7 +93,9 @@ static const acl_perm_map_t acl_nfs4_flag_map[] = { {ARCHIVE_ENTRY_ACL_ENTRY_INHERIT_ONLY, ACL_ENTRY_INHERIT_ONLY}, {ARCHIVE_ENTRY_ACL_ENTRY_SUCCESSFUL_ACCESS, ACL_ENTRY_SUCCESSFUL_ACCESS}, {ARCHIVE_ENTRY_ACL_ENTRY_FAILED_ACCESS, ACL_ENTRY_FAILED_ACCESS}, +#ifdef ACL_ENTRY_INHERITED {ARCHIVE_ENTRY_ACL_ENTRY_INHERITED, ACL_ENTRY_INHERITED} +#endif }; static const int acl_nfs4_flag_map_size = diff --git a/Utilities/cmlibarchive/libarchive/archive_entry.c b/Utilities/cmlibarchive/libarchive/archive_entry.c index 30fb456..f722bbe 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry.c +++ b/Utilities/cmlibarchive/libarchive/archive_entry.c @@ -1491,7 +1491,7 @@ archive_entry_acl_next(struct archive_entry *entry, int want_type, int *type, * the style of the generated ACL. */ wchar_t * -archive_entry_acl_to_text_w(struct archive_entry *entry, ssize_t *len, +archive_entry_acl_to_text_w(struct archive_entry *entry, la_ssize_t *len, int flags) { return (archive_acl_to_text_w(&entry->acl, len, flags, @@ -1499,7 +1499,7 @@ archive_entry_acl_to_text_w(struct archive_entry *entry, ssize_t *len, } char * -archive_entry_acl_to_text(struct archive_entry *entry, ssize_t *len, +archive_entry_acl_to_text(struct archive_entry *entry, la_ssize_t *len, int flags) { return (archive_acl_to_text_l(&entry->acl, len, flags, NULL)); diff --git a/Utilities/cmlibarchive/libarchive/archive_entry.h b/Utilities/cmlibarchive/libarchive/archive_entry.h index 8c8e75a..a9134f6 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry.h +++ b/Utilities/cmlibarchive/libarchive/archive_entry.h @@ -30,7 +30,7 @@ #define ARCHIVE_ENTRY_H_INCLUDED /* Note: Compiler will complain if this does not match archive.h! */ -#define ARCHIVE_VERSION_NUMBER 3003002 +#define ARCHIVE_VERSION_NUMBER 3003003 /* * Note: archive_entry.h is for use outside of libarchive; the @@ -42,6 +42,7 @@ #include <sys/types.h> #include <stddef.h> /* for wchar_t */ +#include <stdint.h> #include <time.h> #if defined(_WIN32) && !defined(__CYGWIN__) diff --git a/Utilities/cmlibarchive/libarchive/archive_match.c b/Utilities/cmlibarchive/libarchive/archive_match.c index be72066..f150e82 100644 --- a/Utilities/cmlibarchive/libarchive/archive_match.c +++ b/Utilities/cmlibarchive/libarchive/archive_match.c @@ -1582,7 +1582,7 @@ time_excluded(struct archive_match *a, struct archive_entry *entry) */ int -archive_match_include_uid(struct archive *_a, int64_t uid) +archive_match_include_uid(struct archive *_a, la_int64_t uid) { struct archive_match *a; @@ -1593,7 +1593,7 @@ archive_match_include_uid(struct archive *_a, int64_t uid) } int -archive_match_include_gid(struct archive *_a, int64_t gid) +archive_match_include_gid(struct archive *_a, la_int64_t gid) { struct archive_match *a; diff --git a/Utilities/cmlibarchive/libarchive/archive_pack_dev.c b/Utilities/cmlibarchive/libarchive/archive_pack_dev.c index 098881b..53bddd7 100644 --- a/Utilities/cmlibarchive/libarchive/archive_pack_dev.c +++ b/Utilities/cmlibarchive/libarchive/archive_pack_dev.c @@ -57,6 +57,9 @@ __RCSID("$NetBSD$"); #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif +#ifdef HAVE_SYS_SYSMACROS_H +#include <sys/sysmacros.h> +#endif #ifdef HAVE_UNISTD_H #include <unistd.h> #endif diff --git a/Utilities/cmlibarchive/libarchive/archive_platform.h b/Utilities/cmlibarchive/libarchive/archive_platform.h index f33208c..e161e64 100644 --- a/Utilities/cmlibarchive/libarchive/archive_platform.h +++ b/Utilities/cmlibarchive/libarchive/archive_platform.h @@ -196,4 +196,10 @@ #define ARCHIVE_ERRNO_MISC (-1) #endif +#if defined(__GNUC__) && (__GNUC__ >= 7) +#define __LA_FALLTHROUGH __attribute__((fallthrough)) +#else +#define __LA_FALLTHROUGH +#endif + #endif /* !ARCHIVE_PLATFORM_H_INCLUDED */ diff --git a/Utilities/cmlibarchive/libarchive/archive_ppmd7.c b/Utilities/cmlibarchive/libarchive/archive_ppmd7.c index 1aed922..d0bacc6 100644 --- a/Utilities/cmlibarchive/libarchive/archive_ppmd7.c +++ b/Utilities/cmlibarchive/libarchive/archive_ppmd7.c @@ -115,14 +115,14 @@ static void Ppmd7_Construct(CPpmd7 *p) memset(p->HB2Flag + 0x40, 8, 0x100 - 0x40); } -static void Ppmd7_Free(CPpmd7 *p, ISzAlloc *alloc) +static void Ppmd7_Free(CPpmd7 *p) { - alloc->Free(alloc, p->Base); + free(p->Base); p->Size = 0; p->Base = 0; } -static Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAlloc *alloc) +static Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size) { if (p->Base == 0 || p->Size != size) { @@ -131,14 +131,14 @@ static Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAlloc *alloc) if (size < UNIT_SIZE) { return False; } - Ppmd7_Free(p, alloc); + Ppmd7_Free(p); p->AlignOffset = #ifdef PPMD_32BIT (4 - size) & 3; #else 4 - (size & 3); #endif - if ((p->Base = (Byte *)alloc->Alloc(alloc, p->AlignOffset + size + if ((p->Base = (Byte *)malloc(p->AlignOffset + size #ifndef PPMD_32BIT + UNIT_SIZE #endif diff --git a/Utilities/cmlibarchive/libarchive/archive_ppmd7_private.h b/Utilities/cmlibarchive/libarchive/archive_ppmd7_private.h index 06c99e8..577d6fb 100644 --- a/Utilities/cmlibarchive/libarchive/archive_ppmd7_private.h +++ b/Utilities/cmlibarchive/libarchive/archive_ppmd7_private.h @@ -95,8 +95,8 @@ typedef struct { /* Base Functions */ void (*Ppmd7_Construct)(CPpmd7 *p); - Bool (*Ppmd7_Alloc)(CPpmd7 *p, UInt32 size, ISzAlloc *alloc); - void (*Ppmd7_Free)(CPpmd7 *p, ISzAlloc *alloc); + Bool (*Ppmd7_Alloc)(CPpmd7 *p, UInt32 size); + void (*Ppmd7_Free)(CPpmd7 *p); void (*Ppmd7_Init)(CPpmd7 *p, unsigned maxOrder); #define Ppmd7_WasAllocated(p) ((p)->Base != NULL) diff --git a/Utilities/cmlibarchive/libarchive/archive_ppmd_private.h b/Utilities/cmlibarchive/libarchive/archive_ppmd_private.h index e78bde5..a83b851 100644 --- a/Utilities/cmlibarchive/libarchive/archive_ppmd_private.h +++ b/Utilities/cmlibarchive/libarchive/archive_ppmd_private.h @@ -69,13 +69,6 @@ typedef struct void (*Write)(void *p, Byte b); } IByteOut; - -typedef struct -{ - void *(*Alloc)(void *p, size_t size); - void (*Free)(void *p, void *address); /* address can be 0 */ -} ISzAlloc; - /*** End defined in Types.h ***/ /*** Begin defined in CpuArch.h ***/ diff --git a/Utilities/cmlibarchive/libarchive/archive_read.c b/Utilities/cmlibarchive/libarchive/archive_read.c index a642a33..0e56e76 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read.c +++ b/Utilities/cmlibarchive/libarchive/archive_read.c @@ -120,7 +120,8 @@ archive_read_new(void) * Record the do-not-extract-to file. This belongs in archive_read_extract.c. */ void -archive_read_extract_set_skip_file(struct archive *_a, int64_t d, int64_t i) +archive_read_extract_set_skip_file(struct archive *_a, la_int64_t d, + la_int64_t i) { struct archive_read *a = (struct archive_read *)_a; @@ -747,7 +748,7 @@ choose_format(struct archive_read *a) * Return the file offset (within the uncompressed data stream) where * the last header started. */ -int64_t +la_int64_t archive_read_header_position(struct archive *_a) { struct archive_read *a = (struct archive_read *)_a; @@ -820,7 +821,7 @@ archive_read_format_capabilities(struct archive *_a) * DO NOT intermingle calls to this function and archive_read_data_block * to read a single entry body. */ -ssize_t +la_ssize_t archive_read_data(struct archive *_a, void *buff, size_t s) { struct archive *a = (struct archive *)_a; @@ -943,7 +944,7 @@ archive_read_data_skip(struct archive *_a) return (r); } -int64_t +la_int64_t archive_seek_data(struct archive *_a, int64_t offset, int whence) { struct archive_read *a = (struct archive_read *)_a; @@ -1626,7 +1627,8 @@ __archive_read_filter_seek(struct archive_read_filter *filter, int64_t offset, switch (whence) { case SEEK_CUR: /* Adjust the offset and use SEEK_SET instead */ - offset += filter->position; + offset += filter->position; + __LA_FALLTHROUGH; case SEEK_SET: cursor = 0; while (1) diff --git a/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c b/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c index 5e4d163..da7c55b 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c @@ -89,6 +89,10 @@ archive_read_append_filter(struct archive *_a, int code) strcpy(str, "lz4"); r1 = archive_read_support_filter_lz4(_a); break; + case ARCHIVE_FILTER_ZSTD: + strcpy(str, "zstd"); + r1 = archive_read_support_filter_zstd(_a); + break; case ARCHIVE_FILTER_LZIP: strcpy(str, "lzip"); r1 = archive_read_support_filter_lzip(_a); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c b/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c index 548ba89..1786cff 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c @@ -127,7 +127,7 @@ archive_read_disk_entry_setup_acls(struct archive_read_disk *a, /* * Enter working directory and return working pathname of archive_entry. * If a pointer to an integer is provided and its value is below zero - * open a file descriptor on this pahtname. + * open a file descriptor on this pathname. */ const char * archive_read_disk_entry_setup_path(struct archive_read_disk *a, diff --git a/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c b/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c index 6961ae6..cdf7541 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c @@ -387,7 +387,7 @@ archive_read_disk_vtable(void) } const char * -archive_read_disk_gname(struct archive *_a, int64_t gid) +archive_read_disk_gname(struct archive *_a, la_int64_t gid) { struct archive_read_disk *a = (struct archive_read_disk *)_a; if (ARCHIVE_OK != __archive_check_magic(_a, ARCHIVE_READ_DISK_MAGIC, @@ -399,7 +399,7 @@ archive_read_disk_gname(struct archive *_a, int64_t gid) } const char * -archive_read_disk_uname(struct archive *_a, int64_t uid) +archive_read_disk_uname(struct archive *_a, la_int64_t uid) { struct archive_read_disk *a = (struct archive_read_disk *)_a; if (ARCHIVE_OK != __archive_check_magic(_a, ARCHIVE_READ_DISK_MAGIC, @@ -413,7 +413,7 @@ archive_read_disk_uname(struct archive *_a, int64_t uid) int archive_read_disk_set_gname_lookup(struct archive *_a, void *private_data, - const char * (*lookup_gname)(void *private, int64_t gid), + const char * (*lookup_gname)(void *private, la_int64_t gid), void (*cleanup_gname)(void *private)) { struct archive_read_disk *a = (struct archive_read_disk *)_a; @@ -432,7 +432,7 @@ archive_read_disk_set_gname_lookup(struct archive *_a, int archive_read_disk_set_uname_lookup(struct archive *_a, void *private_data, - const char * (*lookup_uname)(void *private, int64_t uid), + const char * (*lookup_uname)(void *private, la_int64_t uid), void (*cleanup_uname)(void *private)) { struct archive_read_disk *a = (struct archive_read_disk *)_a; diff --git a/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c b/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c index 3b90330..d82048d 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c @@ -117,7 +117,7 @@ struct filesystem { */ #define MAX_OVERLAPPED 8 -#define BUFFER_SIZE (1024 * 8) +#define READ_BUFFER_SIZE (1024 * 64) /* Default to 64KB per https://technet.microsoft.com/en-us/library/cc938632.aspx */ #define DIRECT_IO 0/* Disabled */ #define ASYNC_IO 1/* Enabled */ @@ -320,7 +320,7 @@ archive_read_disk_vtable(void) } const char * -archive_read_disk_gname(struct archive *_a, int64_t gid) +archive_read_disk_gname(struct archive *_a, la_int64_t gid) { struct archive_read_disk *a = (struct archive_read_disk *)_a; if (ARCHIVE_OK != __archive_check_magic(_a, ARCHIVE_READ_DISK_MAGIC, @@ -332,7 +332,7 @@ archive_read_disk_gname(struct archive *_a, int64_t gid) } const char * -archive_read_disk_uname(struct archive *_a, int64_t uid) +archive_read_disk_uname(struct archive *_a, la_int64_t uid) { struct archive_read_disk *a = (struct archive_read_disk *)_a; if (ARCHIVE_OK != __archive_check_magic(_a, ARCHIVE_READ_DISK_MAGIC, @@ -346,7 +346,7 @@ archive_read_disk_uname(struct archive *_a, int64_t uid) int archive_read_disk_set_gname_lookup(struct archive *_a, void *private_data, - const char * (*lookup_gname)(void *private, int64_t gid), + const char * (*lookup_gname)(void *private, la_int64_t gid), void (*cleanup_gname)(void *private)) { struct archive_read_disk *a = (struct archive_read_disk *)_a; @@ -567,7 +567,7 @@ start_next_async_read(struct archive_read_disk *a, struct tree *t) /* Allocate read buffer. */ if (olp->buff == NULL) { void *p; - size_t s = (size_t)align_num_per_sector(t, BUFFER_SIZE); + size_t s = (size_t)align_num_per_sector(t, READ_BUFFER_SIZE); p = VirtualAlloc(NULL, s, MEM_COMMIT, PAGE_READWRITE); if (p == NULL) { archive_set_error(&a->archive, ENOMEM, @@ -683,7 +683,7 @@ _archive_read_data_block(struct archive *_a, const void **buff, break; } while (r == ARCHIVE_OK && t->ol_num_doing < MAX_OVERLAPPED); } else { - if (start_next_async_read(a, t) == ARCHIVE_FATAL) + if ((r = start_next_async_read(a, t)) == ARCHIVE_FATAL) goto abort_read_data; } @@ -923,6 +923,7 @@ next_entry(struct archive_read_disk *a, struct tree *t, t->entry_fh = CreateFileW(tree_current_access_path(t), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, flags, NULL); if (t->entry_fh == INVALID_HANDLE_VALUE) { + la_dosmaperr(GetLastError()); archive_set_error(&a->archive, errno, "Couldn't open %ls", tree_current_path(a->tree)); return (ARCHIVE_FAILED); @@ -2275,10 +2276,10 @@ setup_sparse_from_disk(struct archive_read_disk *a, if (range.Length.QuadPart > 0) continue; } else { - /* The remaining data is hole. */ + /* The entire file is a hole. Add one data block of size 0 at the end. */ archive_entry_sparse_add_entry(entry, - range.FileOffset.QuadPart, - range.Length.QuadPart); + entry_size, + 0); } break; } else { diff --git a/Utilities/cmlibarchive/libarchive/archive_read_filter.3 b/Utilities/cmlibarchive/libarchive/archive_read_filter.3 index 7f020e3..ef0a701 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_filter.3 +++ b/Utilities/cmlibarchive/libarchive/archive_read_filter.3 @@ -38,6 +38,7 @@ .Nm archive_read_support_filter_rpm , .Nm archive_read_support_filter_uu , .Nm archive_read_support_filter_xz , +.Nm archive_read_support_filter_zstd , .Nm archive_read_support_filter_program , .Nm archive_read_support_filter_program_signature .Nd functions for reading streaming archives @@ -73,6 +74,8 @@ Streaming Archive Library (libarchive, -larchive) .Ft int .Fn archive_read_support_filter_xz "struct archive *" .Ft int +.Fn archive_read_support_filter_zstd "struct archive *" +.Ft int .Fo archive_read_support_filter_program .Fa "struct archive *" .Fa "const char *cmd" @@ -99,7 +102,8 @@ Streaming Archive Library (libarchive, -larchive) .Fn archive_read_support_filter_none , .Fn archive_read_support_filter_rpm , .Fn archive_read_support_filter_uu , -.Fn archive_read_support_filter_xz +.Fn archive_read_support_filter_xz , +.Fn archive_read_support_filter_zstd , .Xc Enables auto-detection code and decompression support for the specified compression. diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c index 68c53de..edb508c 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c @@ -71,6 +71,8 @@ archive_read_support_filter_all(struct archive *a) archive_read_support_filter_grzip(a); /* Lz4 falls back to "lz4 -d" command-line program. */ archive_read_support_filter_lz4(a); + /* Zstd falls back to "zstd -d" command-line program. */ + archive_read_support_filter_zstd(a); /* Note: We always return ARCHIVE_OK here, even if some of the * above return ARCHIVE_WARN. The intent here is to enable diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_zstd.c b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_zstd.c new file mode 100644 index 0000000..c8bb36b --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_zstd.c @@ -0,0 +1,292 @@ +/*- + * Copyright (c) 2009-2011 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" + +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#include <stdio.h> +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#if HAVE_ZSTD_H +#include <zstd.h> +#endif + +#include "archive.h" +#include "archive_endian.h" +#include "archive_private.h" +#include "archive_read_private.h" + +#if HAVE_ZSTD_H && HAVE_LIBZSTD + +struct private_data { + ZSTD_DStream *dstream; + unsigned char *out_block; + size_t out_block_size; + int64_t total_out; + char in_frame; /* True = in the middle of a zstd frame. */ + char eof; /* True = found end of compressed data. */ +}; + +/* Zstd Filter. */ +static ssize_t zstd_filter_read(struct archive_read_filter *, const void**); +static int zstd_filter_close(struct archive_read_filter *); +#endif + +/* + * Note that we can detect zstd compressed files even if we can't decompress + * them. (In fact, we like detecting them because we can give better error + * messages.) So the bid framework here gets compiled even if no zstd library + * is available. + */ +static int zstd_bidder_bid(struct archive_read_filter_bidder *, + struct archive_read_filter *); +static int zstd_bidder_init(struct archive_read_filter *); + +int +archive_read_support_filter_zstd(struct archive *_a) +{ + struct archive_read *a = (struct archive_read *)_a; + struct archive_read_filter_bidder *bidder; + + archive_check_magic(_a, ARCHIVE_READ_MAGIC, + ARCHIVE_STATE_NEW, "archive_read_support_filter_zstd"); + + if (__archive_read_get_bidder(a, &bidder) != ARCHIVE_OK) + return (ARCHIVE_FATAL); + + bidder->data = NULL; + bidder->name = "zstd"; + bidder->bid = zstd_bidder_bid; + bidder->init = zstd_bidder_init; + bidder->options = NULL; + bidder->free = NULL; +#if HAVE_ZSTD_H && HAVE_LIBZSTD + return (ARCHIVE_OK); +#else + archive_set_error(_a, ARCHIVE_ERRNO_MISC, + "Using external zstd program for zstd decompression"); + return (ARCHIVE_WARN); +#endif +} + +/* + * Test whether we can handle this data. + */ +static int +zstd_bidder_bid(struct archive_read_filter_bidder *self, + struct archive_read_filter *filter) +{ + const unsigned char *buffer; + ssize_t avail; + unsigned prefix; + + /* Zstd frame magic values */ + const unsigned zstd_magic = 0xFD2FB528U; + + (void) self; /* UNUSED */ + + buffer = __archive_read_filter_ahead(filter, 4, &avail); + if (buffer == NULL) + return (0); + + prefix = archive_le32dec(buffer); + if (prefix == zstd_magic) + return (32); + + return (0); +} + +#if !(HAVE_ZSTD_H && HAVE_LIBZSTD) + +/* + * If we don't have the library on this system, we can't do the + * decompression directly. We can, however, try to run "zstd -d" + * in case that's available. + */ +static int +zstd_bidder_init(struct archive_read_filter *self) +{ + int r; + + r = __archive_read_program(self, "zstd -d -qq"); + /* Note: We set the format here even if __archive_read_program() + * above fails. We do, after all, know what the format is + * even if we weren't able to read it. */ + self->code = ARCHIVE_FILTER_ZSTD; + self->name = "zstd"; + return (r); +} + +#else + +/* + * Initialize the filter object + */ +static int +zstd_bidder_init(struct archive_read_filter *self) +{ + struct private_data *state; + const size_t out_block_size = ZSTD_DStreamOutSize(); + void *out_block; + ZSTD_DStream *dstream; + + self->code = ARCHIVE_FILTER_ZSTD; + self->name = "zstd"; + + state = (struct private_data *)calloc(sizeof(*state), 1); + out_block = (unsigned char *)malloc(out_block_size); + dstream = ZSTD_createDStream(); + + if (state == NULL || out_block == NULL || dstream == NULL) { + free(out_block); + free(state); + ZSTD_freeDStream(dstream); /* supports free on NULL */ + archive_set_error(&self->archive->archive, ENOMEM, + "Can't allocate data for zstd decompression"); + return (ARCHIVE_FATAL); + } + + self->data = state; + + state->out_block_size = out_block_size; + state->out_block = out_block; + state->dstream = dstream; + self->read = zstd_filter_read; + self->skip = NULL; /* not supported */ + self->close = zstd_filter_close; + + state->eof = 0; + state->in_frame = 0; + + return (ARCHIVE_OK); +} + +static ssize_t +zstd_filter_read(struct archive_read_filter *self, const void **p) +{ + struct private_data *state; + size_t decompressed; + ssize_t avail_in; + ZSTD_outBuffer out; + ZSTD_inBuffer in; + + state = (struct private_data *)self->data; + + out = (ZSTD_outBuffer) { state->out_block, state->out_block_size, 0 }; + + /* Try to fill the output buffer. */ + while (out.pos < out.size && !state->eof) { + if (!state->in_frame) { + const size_t ret = ZSTD_initDStream(state->dstream); + if (ZSTD_isError(ret)) { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Error initializing zstd decompressor: %s", + ZSTD_getErrorName(ret)); + return (ARCHIVE_FATAL); + } + } + in.src = __archive_read_filter_ahead(self->upstream, 1, + &avail_in); + if (avail_in < 0) { + return avail_in; + } + if (in.src == NULL && avail_in == 0) { + if (!state->in_frame) { + /* end of stream */ + state->eof = 1; + break; + } else { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Truncated zstd input"); + return (ARCHIVE_FATAL); + } + } + in.size = avail_in; + in.pos = 0; + + { + const size_t ret = + ZSTD_decompressStream(state->dstream, &out, &in); + + if (ZSTD_isError(ret)) { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Zstd decompression failed: %s", + ZSTD_getErrorName(ret)); + return (ARCHIVE_FATAL); + } + + /* Decompressor made some progress */ + __archive_read_filter_consume(self->upstream, in.pos); + + /* ret guaranteed to be > 0 if frame isn't done yet */ + state->in_frame = (ret != 0); + } + } + + decompressed = out.pos; + state->total_out += decompressed; + if (decompressed == 0) + *p = NULL; + else + *p = state->out_block; + return (decompressed); +} + +/* + * Clean up the decompressor. + */ +static int +zstd_filter_close(struct archive_read_filter *self) +{ + struct private_data *state; + + state = (struct private_data *)self->data; + + ZSTD_freeDStream(state->dstream); + free(state->out_block); + free(state); + + return (ARCHIVE_OK); +} + +#endif /* HAVE_ZLIB_H && HAVE_LIBZSTD */ diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c index 96774f4..a885a4c 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c @@ -975,18 +975,6 @@ decode_codec_id(const unsigned char *codecId, size_t id_size) return (id); } -static void * -ppmd_alloc(void *p, size_t size) -{ - (void)p; - return malloc(size); -} -static void -ppmd_free(void *p, void *address) -{ - (void)p; - free(address); -} static Byte ppmd_read(void *p) { @@ -1006,8 +994,6 @@ ppmd_read(void *p) return (b); } -static ISzAlloc g_szalloc = { ppmd_alloc, ppmd_free }; - static int init_decompression(struct archive_read *a, struct _7zip *zip, const struct _7z_coder *coder1, const struct _7z_coder *coder2) @@ -1237,7 +1223,7 @@ init_decompression(struct archive_read *a, struct _7zip *zip, if (zip->ppmd7_valid) { __archive_ppmd7_functions.Ppmd7_Free( - &zip->ppmd7_context, &g_szalloc); + &zip->ppmd7_context); zip->ppmd7_valid = 0; } @@ -1256,7 +1242,7 @@ init_decompression(struct archive_read *a, struct _7zip *zip, } __archive_ppmd7_functions.Ppmd7_Construct(&zip->ppmd7_context); r = __archive_ppmd7_functions.Ppmd7_Alloc( - &zip->ppmd7_context, msize, &g_szalloc); + &zip->ppmd7_context, msize); if (r == 0) { archive_set_error(&a->archive, ENOMEM, "Coludn't allocate memory for PPMd"); @@ -1636,7 +1622,7 @@ free_decompression(struct archive_read *a, struct _7zip *zip) #endif if (zip->ppmd7_valid) { __archive_ppmd7_functions.Ppmd7_Free( - &zip->ppmd7_context, &g_szalloc); + &zip->ppmd7_context); zip->ppmd7_valid = 0; } return (r); @@ -2569,6 +2555,7 @@ read_Header(struct archive_read *a, struct _7z_header_info *h, case kDummy: if (ll == 0) break; + __LA_FALLTHROUGH; default: if (header_bytes(a, ll) == NULL) return (-1); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c index ad9f782..67d5b21 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c @@ -633,6 +633,13 @@ header_newc(struct archive_read *a, struct cpio *cpio, /* Pad name to 2 more than a multiple of 4. */ *name_pad = (2 - *namelength) & 3; + /* Make sure that the padded name length fits into size_t. */ + if (*name_pad > SIZE_MAX - *namelength) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "cpio archive has invalid namelength"); + return (ARCHIVE_FATAL); + } + /* * Note: entry_bytes_remaining is at least 64 bits and * therefore guaranteed to be big enough for a 33-bit file diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c index 4b436d1..93649f8 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c @@ -409,7 +409,8 @@ static int next_entry_seek(struct archive_read *, struct iso9660 *, struct file_info **); static struct file_info * parse_file_info(struct archive_read *a, - struct file_info *parent, const unsigned char *isodirrec); + struct file_info *parent, const unsigned char *isodirrec, + size_t reclen); static int parse_rockridge(struct archive_read *a, struct file_info *file, const unsigned char *start, const unsigned char *end); @@ -1022,7 +1023,7 @@ read_children(struct archive_read *a, struct file_info *parent) if (*(p + DR_name_len_offset) == 1 && *(p + DR_name_offset) == '\001') continue; - child = parse_file_info(a, parent, p); + child = parse_file_info(a, parent, p, b - p); if (child == NULL) { __archive_read_consume(a, skip_size); return (ARCHIVE_FATAL); @@ -1112,7 +1113,7 @@ choose_volume(struct archive_read *a, struct iso9660 *iso9660) */ seenJoliet = iso9660->seenJoliet;/* Save flag. */ iso9660->seenJoliet = 0; - file = parse_file_info(a, NULL, block); + file = parse_file_info(a, NULL, block, vd->size); if (file == NULL) return (ARCHIVE_FATAL); iso9660->seenJoliet = seenJoliet; @@ -1144,7 +1145,7 @@ choose_volume(struct archive_read *a, struct iso9660 *iso9660) return (ARCHIVE_FATAL); } iso9660->seenJoliet = 0; - file = parse_file_info(a, NULL, block); + file = parse_file_info(a, NULL, block, vd->size); if (file == NULL) return (ARCHIVE_FATAL); iso9660->seenJoliet = seenJoliet; @@ -1749,7 +1750,7 @@ archive_read_format_iso9660_cleanup(struct archive_read *a) */ static struct file_info * parse_file_info(struct archive_read *a, struct file_info *parent, - const unsigned char *isodirrec) + const unsigned char *isodirrec, size_t reclen) { struct iso9660 *iso9660; struct file_info *file, *filep; @@ -1763,16 +1764,20 @@ parse_file_info(struct archive_read *a, struct file_info *parent, iso9660 = (struct iso9660 *)(a->format->data); - dr_len = (size_t)isodirrec[DR_length_offset]; - name_len = (size_t)isodirrec[DR_name_len_offset]; - location = archive_le32dec(isodirrec + DR_extent_offset); - fsize = toi(isodirrec + DR_size_offset, DR_size_size); - /* Sanity check that dr_len needs at least 34. */ - if (dr_len < 34) { + if (reclen != 0) + dr_len = (size_t)isodirrec[DR_length_offset]; + /* + * Sanity check that reclen is not zero and dr_len is greater than + * reclen but at least 34 + */ + if (reclen == 0 || reclen < dr_len || dr_len < 34) { archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Invalid length of directory record"); + "Invalid length of directory record"); return (NULL); } + name_len = (size_t)isodirrec[DR_name_len_offset]; + location = archive_le32dec(isodirrec + DR_extent_offset); + fsize = toi(isodirrec + DR_size_offset, DR_size_size); /* Sanity check that name_len doesn't exceed dr_len. */ if (dr_len - 33 < name_len || name_len == 0) { archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c index 1995e9c..98d02c6 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c @@ -701,6 +701,12 @@ archive_read_format_lha_read_header(struct archive_read *a, * Prepare variables used to read a file content. */ lha->entry_bytes_remaining = lha->compsize; + if (lha->entry_bytes_remaining < 0) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, + "Invalid LHa entry size"); + return (ARCHIVE_FATAL); + } lha->entry_offset = 0; lha->entry_crc_calculated = 0; diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c index 44b6083..5b0eadc 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c @@ -49,6 +49,7 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_read_support_format_mtree.c 2011 #include "archive.h" #include "archive_entry.h" #include "archive_private.h" +#include "archive_rb.h" #include "archive_read_private.h" #include "archive_string.h" #include "archive_pack_dev.h" @@ -75,7 +76,7 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_read_support_format_mtree.c 2011 #define MTREE_HAS_OPTIONAL 0x0800 #define MTREE_HAS_NOCHANGE 0x1000 /* FreeBSD specific */ -#define MTREE_HASHTABLE_SIZE 1024 +#define MAX_LINE_LEN (1024 * 1024) struct mtree_option { struct mtree_option *next; @@ -83,13 +84,13 @@ struct mtree_option { }; struct mtree_entry { + struct archive_rb_node rbnode; + struct mtree_entry *next_dup; struct mtree_entry *next; struct mtree_option *options; char *name; char full; char used; - unsigned int name_hash; - struct mtree_entry *hashtable_next; }; struct mtree { @@ -102,11 +103,12 @@ struct mtree { const char *archive_format_name; struct mtree_entry *entries; struct mtree_entry *this_entry; - struct mtree_entry *entry_hashtable[MTREE_HASHTABLE_SIZE]; + struct archive_rb_tree entry_rbtree; struct archive_string current_dir; struct archive_string contents_name; struct archive_entry_linkresolver *resolver; + struct archive_rb_tree rbtree; int64_t cur_size; char checkfs; @@ -115,7 +117,6 @@ struct mtree { static int bid_keycmp(const char *, const char *, ssize_t); static int cleanup(struct archive_read *); static int detect_form(struct archive_read *, int *); -static unsigned int hash(const char *); static int mtree_bid(struct archive_read *, int); static int parse_file(struct archive_read *, struct archive_entry *, struct mtree *, struct mtree_entry *, int *); @@ -217,9 +218,30 @@ free_options(struct mtree_option *head) } } +static int +mtree_cmp_node(const struct archive_rb_node *n1, + const struct archive_rb_node *n2) +{ + const struct mtree_entry *e1 = (const struct mtree_entry *)n1; + const struct mtree_entry *e2 = (const struct mtree_entry *)n2; + + return (strcmp(e1->name, e2->name)); +} + +static int +mtree_cmp_key(const struct archive_rb_node *n, const void *key) +{ + const struct mtree_entry *e = (const struct mtree_entry *)n; + + return (strcmp(e->name, key)); +} + int archive_read_support_format_mtree(struct archive *_a) { + static const struct archive_rb_tree_ops rb_ops = { + mtree_cmp_node, mtree_cmp_key, + }; struct archive_read *a = (struct archive_read *)_a; struct mtree *mtree; int r; @@ -235,6 +257,8 @@ archive_read_support_format_mtree(struct archive *_a) } mtree->fd = -1; + __archive_rb_tree_init(&mtree->rbtree, &rb_ops); + r = __archive_read_register_format(a, mtree, "mtree", mtree_bid, archive_read_format_mtree_options, read_header, read_data, skip, NULL, cleanup, NULL, NULL); @@ -334,6 +358,14 @@ next_line(struct archive_read *a, size_t nbytes_req = (*ravail+1023) & ~1023U; ssize_t tested; + /* + * Place an arbitrary limit on the line length. + * mtree is almost free-form input and without line length limits, + * it can consume a lot of memory. + */ + if (len >= MAX_LINE_LEN) + return (-1); + /* Increase reading bytes if it is not enough to at least * new two lines. */ if (nbytes_req < (size_t)*ravail + 160) @@ -865,12 +897,11 @@ process_add_entry(struct archive_read *a, struct mtree *mtree, struct mtree_option **global, const char *line, ssize_t line_len, struct mtree_entry **last_entry, int is_form_d) { - struct mtree_entry *entry, *ht_iter; + struct mtree_entry *entry; struct mtree_option *iter; const char *next, *eq, *name, *end; size_t name_len, len; int r, i; - unsigned int ht_idx; if ((entry = malloc(sizeof(*entry))) == NULL) { archive_set_error(&a->archive, errno, "Can't allocate memory"); @@ -881,8 +912,6 @@ process_add_entry(struct archive_read *a, struct mtree *mtree, entry->name = NULL; entry->used = 0; entry->full = 0; - entry->name_hash = 0; - entry->hashtable_next = NULL; /* Add this entry to list. */ if (*last_entry == NULL) @@ -935,15 +964,17 @@ process_add_entry(struct archive_read *a, struct mtree *mtree, memcpy(entry->name, name, name_len); entry->name[name_len] = '\0'; parse_escapes(entry->name, entry); - entry->name_hash = hash(entry->name); - ht_idx = entry->name_hash % MTREE_HASHTABLE_SIZE; - if ((ht_iter = mtree->entry_hashtable[ht_idx]) != NULL) { - while (ht_iter->hashtable_next) - ht_iter = ht_iter->hashtable_next; - ht_iter->hashtable_next = entry; - } else { - mtree->entry_hashtable[ht_idx] = entry; + entry->next_dup = NULL; + if (entry->full) { + if (!__archive_rb_tree_insert_node(&mtree->rbtree, &entry->rbnode)) { + struct mtree_entry *alt; + alt = (struct mtree_entry *)__archive_rb_tree_find_node( + &mtree->rbtree, entry->name); + while (alt->next_dup) + alt = alt->next_dup; + alt->next_dup = entry; + } } for (iter = *global; iter != NULL; iter = iter->next) { @@ -1138,14 +1169,13 @@ parse_file(struct archive_read *a, struct archive_entry *entry, * with pathname canonicalization, which is a very * tricky subject.) */ - for (mp = mentry->hashtable_next; mp != NULL; mp = mp->hashtable_next) { - if (mp->full && !mp->used - && mentry->name_hash == mp->name_hash - && strcmp(mentry->name, mp->name) == 0) { + mp = (struct mtree_entry *)__archive_rb_tree_find_node( + &mtree->rbtree, mentry->name); + for (; mp; mp = mp->next_dup) { + if (mp->full && !mp->used) { /* Later lines override earlier ones. */ mp->used = 1; - r1 = parse_line(a, entry, mtree, mp, - &parsed_kws); + r1 = parse_line(a, entry, mtree, mp, &parsed_kws); if (r1 < r) r = r1; } @@ -1489,6 +1519,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, } if (strcmp(key, "cksum") == 0) break; + __LA_FALLTHROUGH; case 'd': if (strcmp(key, "device") == 0) { /* stat(2) st_rdev field, e.g. the major/minor IDs @@ -1502,12 +1533,14 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, archive_entry_set_rdev(entry, dev); return r; } + __LA_FALLTHROUGH; case 'f': if (strcmp(key, "flags") == 0) { *parsed_kws |= MTREE_HAS_FFLAGS; archive_entry_copy_fflags_text(entry, val); break; } + __LA_FALLTHROUGH; case 'g': if (strcmp(key, "gid") == 0) { *parsed_kws |= MTREE_HAS_GID; @@ -1519,16 +1552,19 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, archive_entry_copy_gname(entry, val); break; } + __LA_FALLTHROUGH; case 'i': if (strcmp(key, "inode") == 0) { archive_entry_set_ino(entry, mtree_atol(&val, 10)); break; } + __LA_FALLTHROUGH; case 'l': if (strcmp(key, "link") == 0) { archive_entry_copy_symlink(entry, val); break; } + __LA_FALLTHROUGH; case 'm': if (strcmp(key, "md5") == 0 || strcmp(key, "md5digest") == 0) break; @@ -1545,6 +1581,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, } break; } + __LA_FALLTHROUGH; case 'n': if (strcmp(key, "nlink") == 0) { *parsed_kws |= MTREE_HAS_NLINK; @@ -1552,6 +1589,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, (unsigned int)mtree_atol(&val, 10)); break; } + __LA_FALLTHROUGH; case 'r': if (strcmp(key, "resdevice") == 0) { /* stat(2) st_dev field, e.g. the device ID where the @@ -1567,6 +1605,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, if (strcmp(key, "rmd160") == 0 || strcmp(key, "rmd160digest") == 0) break; + __LA_FALLTHROUGH; case 's': if (strcmp(key, "sha1") == 0 || strcmp(key, "sha1digest") == 0) break; @@ -1583,6 +1622,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, archive_entry_set_size(entry, mtree_atol(&val, 10)); break; } + __LA_FALLTHROUGH; case 't': if (strcmp(key, "tags") == 0) { /* @@ -1625,18 +1665,21 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, archive_entry_set_filetype(entry, AE_IFBLK); break; } + __LA_FALLTHROUGH; case 'c': if (strcmp(val, "char") == 0) { archive_entry_set_filetype(entry, AE_IFCHR); break; } + __LA_FALLTHROUGH; case 'd': if (strcmp(val, "dir") == 0) { archive_entry_set_filetype(entry, AE_IFDIR); break; } + __LA_FALLTHROUGH; case 'f': if (strcmp(val, "fifo") == 0) { archive_entry_set_filetype(entry, @@ -1648,12 +1691,14 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, AE_IFREG); break; } + __LA_FALLTHROUGH; case 'l': if (strcmp(val, "link") == 0) { archive_entry_set_filetype(entry, AE_IFLNK); break; } + __LA_FALLTHROUGH; default: archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, @@ -1665,6 +1710,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, *parsed_kws |= MTREE_HAS_TYPE; break; } + __LA_FALLTHROUGH; case 'u': if (strcmp(key, "uid") == 0) { *parsed_kws |= MTREE_HAS_UID; @@ -1676,6 +1722,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, archive_entry_copy_uname(entry, val); break; } + __LA_FALLTHROUGH; default: archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, "Unrecognized key %s=%s", key, val); @@ -1962,19 +2009,3 @@ readline(struct archive_read *a, struct mtree *mtree, char **start, find_off = u - mtree->line.s; } } - -static unsigned int -hash(const char *p) -{ - /* A 32-bit version of Peter Weinberger's (PJW) hash algorithm, - as used by ELF for hashing function names. */ - unsigned g, h = 0; - while (*p != '\0') { - h = (h << 4) + *p++; - if ((g = h & 0xF0000000) != 0) { - h ^= g >> 24; - h &= 0x0FFFFFFF; - } - } - return h; -} diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c index a0e641e..c68d77b 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c @@ -604,20 +604,6 @@ lzss_emit_match(struct rar *rar, int offset, int length) rar->lzss.position += length; } -static void * -ppmd_alloc(void *p, size_t size) -{ - (void)p; - return malloc(size); -} -static void -ppmd_free(void *p, void *address) -{ - (void)p; - free(address); -} -static ISzAlloc g_szalloc = { ppmd_alloc, ppmd_free }; - static Byte ppmd_read(void *p) { @@ -1038,7 +1024,7 @@ archive_read_format_rar_read_data(struct archive_read *a, const void **buff, case COMPRESS_METHOD_BEST: ret = read_data_compressed(a, buff, size, offset); if (ret != ARCHIVE_OK && ret != ARCHIVE_WARN) - __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context, &g_szalloc); + __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context); break; default: @@ -1253,7 +1239,7 @@ archive_read_format_rar_cleanup(struct archive_read *a) free(rar->dbo); free(rar->unp_buffer); free(rar->lzss.window); - __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context, &g_szalloc); + __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context); free(rar); (a->format->data) = NULL; return (ARCHIVE_OK); @@ -1496,7 +1482,11 @@ read_header(struct archive_read *a, struct archive_entry *entry, return (ARCHIVE_FATAL); } filename[filename_size++] = '\0'; - filename[filename_size++] = '\0'; + /* + * Do not increment filename_size here as the computations below + * add the space for the terminating NUL explicitly. + */ + filename[filename_size] = '\0'; /* Decoded unicode form is UTF-16BE, so we have to update a string * conversion object for it. */ @@ -1654,7 +1644,7 @@ read_header(struct archive_read *a, struct archive_entry *entry, rar->unp_offset = 0; rar->unp_buffer_size = UNP_BUFFER_SIZE; memset(rar->lengthtable, 0, sizeof(rar->lengthtable)); - __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context, &g_szalloc); + __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context); rar->ppmd_valid = rar->ppmd_eod = 0; /* Don't set any archive entries for non-file header types */ @@ -2118,7 +2108,7 @@ parse_codes(struct archive_read *a) /* Make sure ppmd7_contest is freed before Ppmd7_Construct * because reading a broken file cause this abnormal sequence. */ - __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context, &g_szalloc); + __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context); rar->bytein.a = a; rar->bytein.Read = &ppmd_read; @@ -2133,7 +2123,7 @@ parse_codes(struct archive_read *a) } if (!__archive_ppmd7_functions.Ppmd7_Alloc(&rar->ppmd7_context, - rar->dictionary_size, &g_szalloc)) + rar->dictionary_size)) { archive_set_error(&a->archive, ENOMEM, "Out of memory"); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c index 30d5bc8..60800bb 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c @@ -251,15 +251,15 @@ archive_read_support_format_tar(struct archive *_a) ARCHIVE_STATE_NEW, "archive_read_support_format_tar"); tar = (struct tar *)calloc(1, sizeof(*tar)); -#ifdef HAVE_COPYFILE_H - /* Set this by default on Mac OS. */ - tar->process_mac_extensions = 1; -#endif if (tar == NULL) { archive_set_error(&a->archive, ENOMEM, "Can't allocate tar data"); return (ARCHIVE_FATAL); } +#ifdef HAVE_COPYFILE_H + /* Set this by default on Mac OS. */ + tar->process_mac_extensions = 1; +#endif r = __archive_read_register_format(a, tar, "tar", archive_read_format_tar_bid, @@ -2241,7 +2241,7 @@ gnu_add_sparse_entry(struct archive_read *a, struct tar *tar, else tar->sparse_list = p; tar->sparse_last = p; - if (remaining < 0 || offset < 0) { + if (remaining < 0 || offset < 0 || offset > INT64_MAX - remaining) { archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "Malformed sparse map data"); return (ARCHIVE_FATAL); } diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c index eeac1c8..9292ed7 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c @@ -1040,6 +1040,9 @@ atol10(const char *p, size_t char_cnt) uint64_t l; int digit; + if (char_cnt == 0) + return (0); + l = 0; digit = *p - '0'; while (digit >= 0 && digit < 10 && char_cnt-- > 0) { @@ -1054,7 +1057,10 @@ atol8(const char *p, size_t char_cnt) { int64_t l; int digit; - + + if (char_cnt == 0) + return (0); + l = 0; while (char_cnt-- > 0) { if (*p >= '0' && *p <= '7') @@ -2623,6 +2629,14 @@ strappend_base64(struct xar *xar, archive_strncat(as, (const char *)buff, len); } +static int +is_string(const char *known, const char *data, size_t len) +{ + if (strlen(known) != len) + return -1; + return memcmp(data, known, len); +} + static void xml_data(void *userData, const char *s, int len) { @@ -2674,26 +2688,26 @@ xml_data(void *userData, const char *s, int len) archive_strncpy(&(xar->file->symlink), s, len); break; case FILE_TYPE: - if (strncmp("file", s, len) == 0 || - strncmp("hardlink", s, len) == 0) + if (is_string("file", s, len) == 0 || + is_string("hardlink", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFREG; - if (strncmp("directory", s, len) == 0) + if (is_string("directory", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFDIR; - if (strncmp("symlink", s, len) == 0) + if (is_string("symlink", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFLNK; - if (strncmp("character special", s, len) == 0) + if (is_string("character special", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFCHR; - if (strncmp("block special", s, len) == 0) + if (is_string("block special", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFBLK; - if (strncmp("socket", s, len) == 0) + if (is_string("socket", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFSOCK; - if (strncmp("fifo", s, len) == 0) + if (is_string("fifo", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFIFO; xar->file->has |= HAS_TYPE; diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c index ddd4458..7e99b12 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c @@ -511,7 +511,13 @@ process_extra(struct archive_read *a, const char *p, size_t extra_length, struct case 0x5455: { /* Extended time field "UT". */ - int flags = p[offset]; + int flags; + if (datasize == 0) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Incomplete extended time field"); + return ARCHIVE_FAILED; + } + flags = p[offset]; offset++; datasize--; /* Flag bits indicate which dates are present. */ @@ -723,6 +729,11 @@ process_extra(struct archive_read *a, const char *p, size_t extra_length, struct } case 0x9901: /* WinZip AES extra data field. */ + if (datasize < 6) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Incomplete AES field"); + return ARCHIVE_FAILED; + } if (p[offset + 2] == 'A' && p[offset + 3] == 'E') { /* Vendor version. */ zip_entry->aes_extra.vendor = @@ -881,6 +892,24 @@ zip_read_local_file_header(struct archive_read *a, struct archive_entry *entry, zip_entry->mode |= 0664; } + /* Windows archivers sometimes use backslash as the directory separator. + Normalize to slash. */ + if (zip_entry->system == 0 && + (wp = archive_entry_pathname_w(entry)) != NULL) { + if (wcschr(wp, L'/') == NULL && wcschr(wp, L'\\') != NULL) { + size_t i; + struct archive_wstring s; + archive_string_init(&s); + archive_wstrcpy(&s, wp); + for (i = 0; i < archive_strlen(&s); i++) { + if (s.s[i] == '\\') + s.s[i] = '/'; + } + archive_entry_copy_pathname_w(entry, s.s); + archive_wstring_free(&s); + } + } + /* Make sure that entries with a trailing '/' are marked as directories * even if the External File Attributes contains bogus values. If this * is not a directory and there is no type, assume regularfile. */ @@ -1056,6 +1085,7 @@ zip_read_local_file_header(struct archive_read *a, struct archive_entry *entry, zip->end_of_entry = 1; /* Set up a more descriptive format name. */ + archive_string_empty(&zip->format_name); archive_string_sprintf(&zip->format_name, "ZIP %d.%d (%s)", version / 10, version % 10, compression_name(zip->entry->compression)); diff --git a/Utilities/cmlibarchive/libarchive/archive_string.c b/Utilities/cmlibarchive/libarchive/archive_string.c index 5ae09b6..554533e 100644 --- a/Utilities/cmlibarchive/libarchive/archive_string.c +++ b/Utilities/cmlibarchive/libarchive/archive_string.c @@ -214,7 +214,8 @@ archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s) { if (archive_wstring_ensure(as, as->length + s + 1) == NULL) return (NULL); - wmemmove(as->s + as->length, p, s); + if (s) + wmemmove(as->s + as->length, p, s); as->length += s; as->s[as->length] = 0; return (as); diff --git a/Utilities/cmlibarchive/libarchive/archive_util.c b/Utilities/cmlibarchive/libarchive/archive_util.c index 1e36ad7..e5c6e3b 100644 --- a/Utilities/cmlibarchive/libarchive/archive_util.c +++ b/Utilities/cmlibarchive/libarchive/archive_util.c @@ -140,7 +140,7 @@ archive_compression_name(struct archive *a) /* * Return a count of the number of compressed bytes processed. */ -int64_t +la_int64_t archive_position_compressed(struct archive *a) { return archive_filter_bytes(a, -1); @@ -149,7 +149,7 @@ archive_position_compressed(struct archive *a) /* * Return a count of the number of uncompressed bytes processed. */ -int64_t +la_int64_t archive_position_uncompressed(struct archive *a) { return archive_filter_bytes(a, 0); diff --git a/Utilities/cmlibarchive/libarchive/archive_version_details.c b/Utilities/cmlibarchive/libarchive/archive_version_details.c index 9289bf1..e773e5e 100644 --- a/Utilities/cmlibarchive/libarchive/archive_version_details.c +++ b/Utilities/cmlibarchive/libarchive/archive_version_details.c @@ -45,6 +45,9 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_util.c 201098 2009-12-28 02:58:1 #ifdef HAVE_LZ4_H #include <lz4.h> #endif +#ifdef HAVE_ZSTD_H +#include <zstd.h> +#endif #include "archive.h" #include "archive_private.h" @@ -59,6 +62,7 @@ archive_version_details(void) const char *liblzma = archive_liblzma_version(); const char *bzlib = archive_bzlib_version(); const char *liblz4 = archive_liblz4_version(); + const char *libzstd = archive_libzstd_version(); if (!init) { archive_string_init(&str); @@ -84,6 +88,10 @@ archive_version_details(void) archive_strcat(&str, " liblz4/"); archive_strcat(&str, liblz4); } + if (libzstd) { + archive_strcat(&str, " libzstd/"); + archive_strcat(&str, libzstd); + } } return str.s; } @@ -131,3 +139,13 @@ archive_liblz4_version(void) return NULL; #endif } + +const char * +archive_libzstd_version(void) +{ +#if HAVE_ZSTD_H && HAVE_LIBZSTD + return ZSTD_VERSION_STRING; +#else + return NULL; +#endif +} diff --git a/Utilities/cmlibarchive/libarchive/archive_virtual.c b/Utilities/cmlibarchive/libarchive/archive_virtual.c index de2595a..f509ee5 100644 --- a/Utilities/cmlibarchive/libarchive/archive_virtual.c +++ b/Utilities/cmlibarchive/libarchive/archive_virtual.c @@ -48,7 +48,7 @@ archive_filter_name(struct archive *a, int n) return ((a->vtable->archive_filter_name)(a, n)); } -int64_t +la_int64_t archive_filter_bytes(struct archive *a, int n) { return ((a->vtable->archive_filter_bytes)(a, n)); @@ -124,14 +124,15 @@ archive_write_finish_entry(struct archive *a) return ((a->vtable->archive_write_finish_entry)(a)); } -ssize_t +la_ssize_t archive_write_data(struct archive *a, const void *buff, size_t s) { return ((a->vtable->archive_write_data)(a, buff, s)); } -ssize_t -archive_write_data_block(struct archive *a, const void *buff, size_t s, int64_t o) +la_ssize_t +archive_write_data_block(struct archive *a, const void *buff, size_t s, + la_int64_t o) { if (a->vtable->archive_write_data_block == NULL) { archive_set_error(a, ARCHIVE_ERRNO_MISC, @@ -156,7 +157,7 @@ archive_read_next_header2(struct archive *a, struct archive_entry *entry) int archive_read_data_block(struct archive *a, - const void **buff, size_t *s, int64_t *o) + const void **buff, size_t *s, la_int64_t *o) { return ((a->vtable->archive_read_data_block)(a, buff, s, o)); } diff --git a/Utilities/cmlibarchive/libarchive/archive_write.3 b/Utilities/cmlibarchive/libarchive/archive_write.3 index 376d71d..c1164f5 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write.3 @@ -71,7 +71,7 @@ support. .\" .Ss Set options See -.Xr archive_read_set_options 3 . +.Xr archive_write_set_options 3 . .\" .Ss Open archive See diff --git a/Utilities/cmlibarchive/libarchive/archive_write.c b/Utilities/cmlibarchive/libarchive/archive_write.c index 0634a22..e8daf53 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write.c +++ b/Utilities/cmlibarchive/libarchive/archive_write.c @@ -190,7 +190,7 @@ archive_write_get_bytes_in_last_block(struct archive *_a) * an archive to itself recursively. */ int -archive_write_set_skip_file(struct archive *_a, int64_t d, int64_t i) +archive_write_set_skip_file(struct archive *_a, la_int64_t d, la_int64_t i) { struct archive_write *a = (struct archive_write *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC, diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c index 08f518a..203f414 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c @@ -53,6 +53,7 @@ struct { int code; int (*setter)(struct archive *); } codes[] = { ARCHIVE_FILTER_LZOP, archive_write_add_filter_lzip }, { ARCHIVE_FILTER_UU, archive_write_add_filter_uuencode }, { ARCHIVE_FILTER_XZ, archive_write_add_filter_xz }, + { ARCHIVE_FILTER_ZSTD, archive_write_add_filter_zstd }, { -1, NULL } }; diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c index 85a8d47..ffa633c 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c @@ -57,6 +57,7 @@ struct { const char *name; int (*setter)(struct archive *); } names[] = { "lzop", archive_write_add_filter_lzop }, { "uuencode", archive_write_add_filter_uuencode }, { "xz", archive_write_add_filter_xz }, + { "zstd", archive_write_add_filter_zstd }, { NULL, NULL } }; diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_gzip.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_gzip.c index bbcc178..986123a 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_gzip.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_gzip.c @@ -226,7 +226,12 @@ archive_compressor_gzip_open(struct archive_write_filter *f) data->compressed[7] = (uint8_t)(t>>24)&0xff; } else memset(&data->compressed[4], 0, 4); - data->compressed[8] = 0; /* No deflate options */ + if (data->compression_level == 9) + data->compressed[8] = 2; + else if(data->compression_level == 1) + data->compressed[8] = 4; + else + data->compressed[8] = 0; data->compressed[9] = 3; /* OS=Unix */ data->stream.next_out += 10; data->stream.avail_out -= 10; diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_zstd.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_zstd.c new file mode 100644 index 0000000..671fc6a --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_zstd.c @@ -0,0 +1,335 @@ +/*- + * Copyright (c) 2017 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" + +__FBSDID("$FreeBSD$"); + + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#ifdef HAVE_ZSTD_H +#include <zstd.h> +#endif + +#include "archive.h" +#include "archive_private.h" +#include "archive_string.h" +#include "archive_write_private.h" + +/* Don't compile this if we don't have zstd.h */ + +struct private_data { + int compression_level; +#if HAVE_ZSTD_H && HAVE_LIBZSTD + ZSTD_CStream *cstream; + int64_t total_in; + ZSTD_outBuffer out; +#else + struct archive_write_program_data *pdata; +#endif +}; + +static int archive_compressor_zstd_options(struct archive_write_filter *, + const char *, const char *); +static int archive_compressor_zstd_open(struct archive_write_filter *); +static int archive_compressor_zstd_write(struct archive_write_filter *, + const void *, size_t); +static int archive_compressor_zstd_close(struct archive_write_filter *); +static int archive_compressor_zstd_free(struct archive_write_filter *); +#if HAVE_ZSTD_H && HAVE_LIBZSTD +static int drive_compressor(struct archive_write_filter *, + struct private_data *, int, const void *, size_t); +#endif + + +/* + * Add a zstd compression filter to this write handle. + */ +int +archive_write_add_filter_zstd(struct archive *_a) +{ + struct archive_write *a = (struct archive_write *)_a; + struct archive_write_filter *f = __archive_write_allocate_filter(_a); + struct private_data *data; + archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC, + ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd"); + + data = calloc(1, sizeof(*data)); + if (data == NULL) { + archive_set_error(&a->archive, ENOMEM, "Out of memory"); + return (ARCHIVE_FATAL); + } + f->data = data; + f->open = &archive_compressor_zstd_open; + f->options = &archive_compressor_zstd_options; + f->close = &archive_compressor_zstd_close; + f->free = &archive_compressor_zstd_free; + f->code = ARCHIVE_FILTER_ZSTD; + f->name = "zstd"; + data->compression_level = 3; /* Default level used by the zstd CLI */ +#if HAVE_ZSTD_H && HAVE_LIBZSTD + data->cstream = ZSTD_createCStream(); + if (data->cstream == NULL) { + free(data); + archive_set_error(&a->archive, ENOMEM, + "Failed to allocate zstd compressor object"); + return (ARCHIVE_FATAL); + } + + return (ARCHIVE_OK); +#else + data->pdata = __archive_write_program_allocate("zstd"); + if (data->pdata == NULL) { + free(data); + archive_set_error(&a->archive, ENOMEM, "Out of memory"); + return (ARCHIVE_FATAL); + } + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Using external zstd program"); + return (ARCHIVE_WARN); +#endif +} + +static int +archive_compressor_zstd_free(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; +#if HAVE_ZSTD_H && HAVE_LIBZSTD + ZSTD_freeCStream(data->cstream); + free(data->out.dst); +#else + __archive_write_program_free(data->pdata); +#endif + free(data); + f->data = NULL; + return (ARCHIVE_OK); +} + +/* + * Set write options. + */ +static int +archive_compressor_zstd_options(struct archive_write_filter *f, const char *key, + const char *value) +{ + struct private_data *data = (struct private_data *)f->data; + + if (strcmp(key, "compression-level") == 0) { + int level = atoi(value); +#if HAVE_ZSTD_H && HAVE_LIBZSTD + if (level < 1 || level > ZSTD_maxCLevel()) { +#else + /* If we don't have the library, hard-code the max level */ + if (level < 1 || level > 22) { +#endif + return (ARCHIVE_WARN); + } + data->compression_level = level; + return (ARCHIVE_OK); + } + + /* Note: The "warn" return is just to inform the options + * supervisor that we didn't handle it. It will generate + * a suitable error if no one used this option. */ + return (ARCHIVE_WARN); +} + +#if HAVE_ZSTD_H && HAVE_LIBZSTD +/* + * Setup callback. + */ +static int +archive_compressor_zstd_open(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + int ret; + + ret = __archive_write_open_filter(f->next_filter); + if (ret != ARCHIVE_OK) + return (ret); + + if (data->out.dst == NULL) { + size_t bs = ZSTD_CStreamOutSize(), bpb; + if (f->archive->magic == ARCHIVE_WRITE_MAGIC) { + /* Buffer size should be a multiple number of + * the of bytes per block for performance. */ + bpb = archive_write_get_bytes_per_block(f->archive); + if (bpb > bs) + bs = bpb; + else if (bpb != 0) + bs -= bs % bpb; + } + data->out.size = bs; + data->out.pos = 0; + data->out.dst + = (unsigned char *)malloc(data->out.size); + if (data->out.dst == NULL) { + archive_set_error(f->archive, ENOMEM, + "Can't allocate data for compression buffer"); + return (ARCHIVE_FATAL); + } + } + + f->write = archive_compressor_zstd_write; + + if (ZSTD_isError(ZSTD_initCStream(data->cstream, + data->compression_level))) { + archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, + "Internal error initializing zstd compressor object"); + return (ARCHIVE_FATAL); + } + + return (ARCHIVE_OK); +} + +/* + * Write data to the compressed stream. + */ +static int +archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + int ret; + + /* Update statistics */ + data->total_in += length; + + if ((ret = drive_compressor(f, data, 0, buff, length)) != ARCHIVE_OK) + return (ret); + + return (ARCHIVE_OK); +} + +/* + * Finish the compression... + */ +static int +archive_compressor_zstd_close(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + int r1, r2; + + /* Finish zstd frame */ + r1 = drive_compressor(f, data, 1, NULL, 0); + + r2 = __archive_write_close_filter(f->next_filter); + + return r1 < r2 ? r1 : r2; +} + +/* + * Utility function to push input data through compressor, + * writing full output blocks as necessary. + * + * Note that this handles both the regular write case (finishing == + * false) and the end-of-archive case (finishing == true). + */ +static int +drive_compressor(struct archive_write_filter *f, + struct private_data *data, int finishing, const void *src, size_t length) +{ + ZSTD_inBuffer in = (ZSTD_inBuffer) { src, length, 0 }; + + for (;;) { + if (data->out.pos == data->out.size) { + const int ret = __archive_write_filter(f->next_filter, + data->out.dst, data->out.size); + if (ret != ARCHIVE_OK) + return (ARCHIVE_FATAL); + data->out.pos = 0; + } + + /* If there's nothing to do, we're done. */ + if (!finishing && in.pos == in.size) + return (ARCHIVE_OK); + + { + const size_t zstdret = !finishing ? + ZSTD_compressStream(data->cstream, &data->out, &in) + : ZSTD_endStream(data->cstream, &data->out); + + if (ZSTD_isError(zstdret)) { + archive_set_error(f->archive, + ARCHIVE_ERRNO_MISC, + "Zstd compression failed: %s", + ZSTD_getErrorName(zstdret)); + return (ARCHIVE_FATAL); + } + + /* If we're finishing, 0 means nothing left to flush */ + if (finishing && zstdret == 0) { + const int ret = __archive_write_filter(f->next_filter, + data->out.dst, data->out.pos); + return (ret); + } + } + } +} + +#else /* HAVE_ZSTD_H && HAVE_LIBZSTD */ + +static int +archive_compressor_zstd_open(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + struct archive_string as; + int r; + + archive_string_init(&as); + archive_string_sprintf(&as, "zstd -%d", data->compression_level); + + f->write = archive_compressor_zstd_write; + r = __archive_write_program_open(f, data->pdata, as.s); + archive_string_free(&as); + return (r); +} + +static int +archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + + return __archive_write_program_write(f, data->pdata, buff, length); +} + +static int +archive_compressor_zstd_close(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + + return __archive_write_program_close(f, data->pdata); +} + +#endif /* HAVE_ZSTD_H && HAVE_LIBZSTD */ diff --git a/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c b/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c index 4a42a3b..affa503 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c @@ -835,7 +835,7 @@ _archive_write_disk_header(struct archive *_a, struct archive_entry *entry) } int -archive_write_disk_set_skip_file(struct archive *_a, int64_t d, int64_t i) +archive_write_disk_set_skip_file(struct archive *_a, la_int64_t d, la_int64_t i) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1786,7 +1786,7 @@ finish_metadata: int archive_write_disk_set_group_lookup(struct archive *_a, void *private_data, - int64_t (*lookup_gid)(void *private, const char *gname, int64_t gid), + la_int64_t (*lookup_gid)(void *private, const char *gname, la_int64_t gid), void (*cleanup_gid)(void *private)) { struct archive_write_disk *a = (struct archive_write_disk *)_a; @@ -1822,7 +1822,7 @@ archive_write_disk_set_user_lookup(struct archive *_a, } int64_t -archive_write_disk_gid(struct archive *_a, const char *name, int64_t id) +archive_write_disk_gid(struct archive *_a, const char *name, la_int64_t id) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1833,7 +1833,7 @@ archive_write_disk_gid(struct archive *_a, const char *name, int64_t id) } int64_t -archive_write_disk_uid(struct archive *_a, const char *name, int64_t id) +archive_write_disk_uid(struct archive *_a, const char *name, la_int64_t id) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1981,6 +1981,10 @@ restore_entry(struct archive_write_disk *a) if ((en == EISDIR || en == EEXIST) && (a->flags & ARCHIVE_EXTRACT_NO_OVERWRITE)) { /* If we're not overwriting, we're done. */ + if (S_ISDIR(a->mode)) { + /* Don't overwrite any settings on existing directories. */ + a->todo = 0; + } archive_entry_unset_size(a->entry); return (ARCHIVE_OK); } diff --git a/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c b/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c index 94b016e..78eda4a 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c @@ -906,7 +906,7 @@ _archive_write_disk_header(struct archive *_a, struct archive_entry *entry) } int -archive_write_disk_set_skip_file(struct archive *_a, int64_t d, int64_t i) +archive_write_disk_set_skip_file(struct archive *_a, la_int64_t d, la_int64_t i) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1156,7 +1156,7 @@ _archive_write_disk_finish_entry(struct archive *_a) int archive_write_disk_set_group_lookup(struct archive *_a, void *private_data, - int64_t (*lookup_gid)(void *private, const char *gname, int64_t gid), + la_int64_t (*lookup_gid)(void *private, const char *gname, la_int64_t gid), void (*cleanup_gid)(void *private)) { struct archive_write_disk *a = (struct archive_write_disk *)_a; @@ -1192,7 +1192,7 @@ archive_write_disk_set_user_lookup(struct archive *_a, } int64_t -archive_write_disk_gid(struct archive *_a, const char *name, int64_t id) +archive_write_disk_gid(struct archive *_a, const char *name, la_int64_t id) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1203,7 +1203,7 @@ archive_write_disk_gid(struct archive *_a, const char *name, int64_t id) } int64_t -archive_write_disk_uid(struct archive *_a, const char *name, int64_t id) +archive_write_disk_uid(struct archive *_a, const char *name, la_int64_t id) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1322,9 +1322,20 @@ restore_entry(struct archive_write_disk *a) } } + if ((en == ENOENT) && (archive_entry_hardlink(a->entry) != NULL)) { + archive_set_error(&a->archive, en, + "Hard-link target '%s' does not exist.", + archive_entry_hardlink(a->entry)); + return (ARCHIVE_FAILED); + } + if ((en == EISDIR || en == EEXIST) && (a->flags & ARCHIVE_EXTRACT_NO_OVERWRITE)) { /* If we're not overwriting, we're done. */ + if (S_ISDIR(a->mode)) { + /* Don't overwrite any settings on existing directories. */ + a->todo = 0; + } archive_entry_unset_size(a->entry); return (ARCHIVE_OK); } diff --git a/Utilities/cmlibarchive/libarchive/archive_write_filter.3 b/Utilities/cmlibarchive/libarchive/archive_write_filter.3 index e1d1891..d6fa071 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_filter.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write_filter.3 @@ -42,7 +42,8 @@ .Nm archive_write_add_filter_none , .Nm archive_write_add_filter_program , .Nm archive_write_add_filter_uuencode , -.Nm archive_write_add_filter_xz +.Nm archive_write_add_filter_xz , +.Nm archive_write_add_filter_zstd , .Nd functions enabling output filters .Sh LIBRARY Streaming Archive Library (libarchive, -larchive) @@ -76,6 +77,8 @@ Streaming Archive Library (libarchive, -larchive) .Fn archive_write_add_filter_uuencode "struct archive *" .Ft int .Fn archive_write_add_filter_xz "struct archive *" +.Ft int +.Fn archive_write_add_filter_zstd "struct archive *" .Sh DESCRIPTION .Bl -tag -width indent .It Xo @@ -89,6 +92,7 @@ Streaming Archive Library (libarchive, -larchive) .Fn archive_write_add_filter_lzma , .Fn archive_write_add_filter_lzop , .Fn archive_write_add_filter_xz , +.Fn archive_write_add_filter_zstd , .Xc The resulting archive will be compressed as specified. Note that the compressed output is always properly blocked. diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_7zip.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_7zip.c index 3fc5a07..2bd4ec4 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format_7zip.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_7zip.c @@ -2095,19 +2095,6 @@ compression_init_encoder_lzma2(struct archive *a, /* * _7_PPMD compressor. */ -static void * -ppmd_alloc(void *p, size_t size) -{ - (void)p; - return malloc(size); -} -static void -ppmd_free(void *p, void *address) -{ - (void)p; - free(address); -} -static ISzAlloc g_szalloc = { ppmd_alloc, ppmd_free }; static void ppmd_write(void *p, Byte b) { @@ -2167,7 +2154,7 @@ compression_init_encoder_ppmd(struct archive *a, archive_le32enc(props+1, msize); __archive_ppmd7_functions.Ppmd7_Construct(&strm->ppmd7_context); r = __archive_ppmd7_functions.Ppmd7_Alloc( - &strm->ppmd7_context, msize, &g_szalloc); + &strm->ppmd7_context, msize); if (r == 0) { free(strm->buff); free(strm); @@ -2243,7 +2230,7 @@ compression_end_ppmd(struct archive *a, struct la_zstream *lastrm) (void)a; /* UNUSED */ strm = (struct ppmd_stream *)lastrm->real_stream; - __archive_ppmd7_functions.Ppmd7_Free(&strm->ppmd7_context, &g_szalloc); + __archive_ppmd7_functions.Ppmd7_Free(&strm->ppmd7_context); free(strm->buff); free(strm); lastrm->real_stream = NULL; diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_ar.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_ar.c index c9771d8..50305cc 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format_ar.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_ar.c @@ -180,7 +180,7 @@ archive_write_ar_header(struct archive_write *a, struct archive_entry *entry) } memset(buff, ' ', 60); - strncpy(&buff[AR_fmag_offset], "`\n", 2); + memcpy(&buff[AR_fmag_offset], "`\n", 2); if (strcmp(pathname, "/") == 0 ) { /* Entry is archive symbol table in GNU format */ @@ -189,7 +189,7 @@ archive_write_ar_header(struct archive_write *a, struct archive_entry *entry) } if (strcmp(pathname, "__.SYMDEF") == 0) { /* Entry is archive symbol table in BSD format */ - strncpy(buff + AR_name_offset, "__.SYMDEF", 9); + memcpy(buff + AR_name_offset, "__.SYMDEF", 9); goto stat; } if (strcmp(pathname, "//") == 0) { @@ -225,7 +225,7 @@ archive_write_ar_header(struct archive_write *a, struct archive_entry *entry) * actually 15 bytes. */ if (strlen(filename) <= 15) { - strncpy(&buff[AR_name_offset], + memcpy(&buff[AR_name_offset], filename, strlen(filename)); buff[AR_name_offset + strlen(filename)] = '/'; } else { @@ -248,7 +248,7 @@ archive_write_ar_header(struct archive_write *a, struct archive_entry *entry) return (ARCHIVE_FATAL); } - strncpy(se, filename, strlen(filename)); + memcpy(se, filename, strlen(filename)); strcpy(se + strlen(filename), "/\n"); ss = strstr(ar->strtab, se); @@ -285,11 +285,11 @@ archive_write_ar_header(struct archive_write *a, struct archive_entry *entry) * archive header. */ if (strlen(filename) <= 16 && strchr(filename, ' ') == NULL) { - strncpy(&buff[AR_name_offset], filename, strlen(filename)); + memcpy(&buff[AR_name_offset], filename, strlen(filename)); buff[AR_name_offset + strlen(filename)] = ' '; } else { - strncpy(buff + AR_name_offset, "#1/", 3); + memcpy(buff + AR_name_offset, "#1/", 3); if (format_decimal(strlen(filename), buff + AR_name_offset + 3, AR_name_size - 3)) { @@ -374,13 +374,14 @@ archive_write_ar_data(struct archive_write *a, const void *buff, size_t s) return (ARCHIVE_WARN); } - ar->strtab = (char *)malloc(s); + ar->strtab = (char *)malloc(s + 1); if (ar->strtab == NULL) { archive_set_error(&a->archive, ENOMEM, "Can't allocate strtab buffer"); return (ARCHIVE_FATAL); } - strncpy(ar->strtab, buff, s); + memcpy(ar->strtab, buff, s); + ar->strtab[s] = '\0'; ar->has_strtab = 1; } diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_pax.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_pax.c index 0eaf733..3cebeae 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format_pax.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_pax.c @@ -1654,7 +1654,7 @@ build_pax_attribute_name(char *dest, const char *src) * GNU PAX Format 1.0 requires the special name, which pattern is: * <dir>/GNUSparseFile.<pid>/<original file name> * - * Since reproducable archives are more important, use 0 as pid. + * Since reproducible archives are more important, use 0 as pid. * * This function is used for only Sparse file, a file type of which * is regular file. diff --git a/Utilities/cmliblzma/CMakeLists.txt b/Utilities/cmliblzma/CMakeLists.txt index bb3b8a7..e9f8826 100644 --- a/Utilities/cmliblzma/CMakeLists.txt +++ b/Utilities/cmliblzma/CMakeLists.txt @@ -6,22 +6,6 @@ include(CheckSymbolExists) include(CheckTypeSize) include(TestBigEndian) -CHECK_C_SOURCE_COMPILES( - "int test (void *restrict x);\nint main (void) {return 0;}" - HAVE_RESTRICT) - -CHECK_C_SOURCE_COMPILES( -"typedef struct abc *d;\nint test (d __restrict x);\nint main (void) {return 0;}" - HAVE___RESTRICT) - -CHECK_C_SOURCE_COMPILES( - "static inline int test (void) {return 0;}\nint main (void) {return test();}" - HAVE_INLINE) - -CHECK_C_SOURCE_COMPILES ( - "static __inline int test (void) {return 0;}\nint main (void) {return test();}" - HAVE___INLINE) - CHECK_INCLUDE_FILE(byteswap.h HAVE_BYTESWAP_H) CHECK_INCLUDE_FILE(inttypes.h HAVE_INTTYPES_H) CHECK_INCLUDE_FILE(limits.h HAVE_LIMITS_H) @@ -95,7 +79,7 @@ CHECK_TYPE_SIZE("unsigned short" SIZE_OF_UNSIGNED_SHORT) CHECK_TYPE_SIZE("unsigned" SIZE_OF_UNSIGNED) CHECK_TYPE_SIZE("unsigned long" SIZE_OF_UNSIGNED_LONG) CHECK_TYPE_SIZE("unsigned long long" SIZE_OF_UNSIGNED_LONG_LONG) -CHECK_TYPE_SIZE("size_t" SIZE_OF_SIZE_T) +CHECK_TYPE_SIZE("size_t" SIZEOF_SIZE_T) CHECK_TYPE_SIZE("__int64" __INT64) CHECK_TYPE_SIZE("unsigned __int64" UNSIGNED___INT64) diff --git a/Utilities/cmliblzma/COPYING b/Utilities/cmliblzma/COPYING index 43c90d0..20e60d5 100644 --- a/Utilities/cmliblzma/COPYING +++ b/Utilities/cmliblzma/COPYING @@ -47,7 +47,7 @@ XZ Utils Licensing naturally it is not legally required. Here is an example of a good notice to put into "about box" or into documentation: - This software includes code from XZ Utils <http://tukaani.org/xz/>. + This software includes code from XZ Utils <https://tukaani.org/xz/>. The following license texts are included in the following files: - COPYING.LGPLv2.1: GNU Lesser General Public License version 2.1 diff --git a/Utilities/cmliblzma/common/common_w32res.rc b/Utilities/cmliblzma/common/common_w32res.rc index fdb88d1..a70de34 100644 --- a/Utilities/cmliblzma/common/common_w32res.rc +++ b/Utilities/cmliblzma/common/common_w32res.rc @@ -17,7 +17,7 @@ #define MY_VERSION LZMA_VERSION_MAJOR,LZMA_VERSION_MINOR,LZMA_VERSION_PATCH,MY_BUILD #define MY_FILENAME MY_NAME MY_SUFFIX -#define MY_COMPANY "The Tukaani Project <http://tukaani.org/>" +#define MY_COMPANY "The Tukaani Project <https://tukaani.org/>" #define MY_PRODUCT PACKAGE_NAME " <" PACKAGE_URL ">" LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US diff --git a/Utilities/cmliblzma/common/sysdefs.h b/Utilities/cmliblzma/common/sysdefs.h index a6edea8..22f487b 100644 --- a/Utilities/cmliblzma/common/sysdefs.h +++ b/Utilities/cmliblzma/common/sysdefs.h @@ -18,6 +18,7 @@ #if defined(_MSC_VER) # pragma warning(push,1) +# pragma warning(disable: 4028) /* formal parameter different from decl */ # pragma warning(disable: 4142) /* benign redefinition of type */ # pragma warning(disable: 4761) /* integral size mismatch in argument */ #endif @@ -124,9 +125,9 @@ // The code currently assumes that size_t is either 32-bit or 64-bit. #ifndef SIZE_MAX -# if SIZE_OF_SIZE_T == 4 +# if SIZEOF_SIZE_T == 4 # define SIZE_MAX UINT32_MAX -# elif SIZE_OF_SIZE_T == 8 +# elif SIZEOF_SIZE_T == 8 # define SIZE_MAX UINT64_MAX # else # error size_t is not 32-bit or 64-bit @@ -175,6 +176,16 @@ typedef unsigned char _Bool; # include <memory.h> #endif +// As of MSVC 2013, inline and restrict are supported with +// non-standard keywords. +#if defined(_WIN32) && defined(_MSC_VER) +# ifndef inline +# define inline __inline +# endif +# ifndef restrict +# define restrict __restrict +# endif +#endif //////////// // Macros // diff --git a/Utilities/cmliblzma/common/tuklib_integer.h b/Utilities/cmliblzma/common/tuklib_integer.h index 5e8262a..5d82685 100644 --- a/Utilities/cmliblzma/common/tuklib_integer.h +++ b/Utilities/cmliblzma/common/tuklib_integer.h @@ -106,6 +106,17 @@ #endif +//////////////////////////////// +// Compiler-specific features // +//////////////////////////////// + +// Newer Intel C compilers require immintrin.h for _bit_scan_reverse() +// and such functions. +#if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1500) +# include <immintrin.h> +#endif + + /////////////////// // Byte swapping // /////////////////// @@ -329,8 +340,8 @@ unaligned_read32le(const uint8_t *buf) static inline void unaligned_write16be(uint8_t *buf, uint16_t num) { - buf[0] = num >> 8; - buf[1] = num; + buf[0] = (uint8_t)(num >> 8); + buf[1] = (uint8_t)num; return; } @@ -338,8 +349,8 @@ unaligned_write16be(uint8_t *buf, uint16_t num) static inline void unaligned_write16le(uint8_t *buf, uint16_t num) { - buf[0] = num; - buf[1] = num >> 8; + buf[0] = (uint8_t)num; + buf[1] = (uint8_t)(num >> 8); return; } @@ -347,10 +358,10 @@ unaligned_write16le(uint8_t *buf, uint16_t num) static inline void unaligned_write32be(uint8_t *buf, uint32_t num) { - buf[0] = num >> 24; - buf[1] = num >> 16; - buf[2] = num >> 8; - buf[3] = num; + buf[0] = (uint8_t)(num >> 24); + buf[1] = (uint8_t)(num >> 16); + buf[2] = (uint8_t)(num >> 8); + buf[3] = (uint8_t)num; return; } @@ -358,10 +369,10 @@ unaligned_write32be(uint8_t *buf, uint32_t num) static inline void unaligned_write32le(uint8_t *buf, uint32_t num) { - buf[0] = num; - buf[1] = num >> 8; - buf[2] = num >> 16; - buf[3] = num >> 24; + buf[0] = (uint8_t)num; + buf[1] = (uint8_t)(num >> 8); + buf[2] = (uint8_t)(num >> 16); + buf[3] = (uint8_t)(num >> 24); return; } diff --git a/Utilities/cmliblzma/config.h.in b/Utilities/cmliblzma/config.h.in index 9c53150..06f7fcb 100644 --- a/Utilities/cmliblzma/config.h.in +++ b/Utilities/cmliblzma/config.h.in @@ -29,7 +29,7 @@ @SIZE_OF_UNSIGNED_CODE@ @SIZE_OF_UNSIGNED_LONG_CODE@ @SIZE_OF_UNSIGNED_LONG_LONG_CODE@ -@SIZE_OF_SIZE_T_CODE@ +@SIZEOF_SIZE_T_CODE@ /* * If we lack int64_t, define it to the first of __int64, int, long, and long long @@ -180,32 +180,6 @@ typedef uint64_t uintmax_t; #cmakedefine uintptr_t @uintptr_t@ - -#cmakedefine HAVE_RESTRICT -#cmakedefine HAVE___RESTRICT - -#cmakedefine HAVE_INLINE -#cmakedefine HAVE___INLINE - -#ifndef HAVE_RESTRICT -# ifdef HAVE___RESTRICT -# define LZMA_RESTRICT __restrict -# else -# define LZMA_RESTRICT -# endif -#else -# define LZMA_RESTRICT restrict -#endif /* HAVE_RESTRICT */ - -#ifndef HAVE_INLINE -# ifdef HAVE___INLINE -# define inline __inline -# else -# define inline -# endif -#endif /* HAVE_INLINE */ - - #cmakedefine WORDS_BIGENDIAN 1 #cmakedefine HAVE_BYTESWAP_H 1 diff --git a/Utilities/cmliblzma/liblzma/api/lzma.h b/Utilities/cmliblzma/liblzma/api/lzma.h index fb874c3..aa88e42 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma.h +++ b/Utilities/cmliblzma/liblzma/api/lzma.h @@ -82,12 +82,20 @@ # if !defined(UINT32_C) || !defined(UINT64_C) \ || !defined(UINT32_MAX) || !defined(UINT64_MAX) /* - * MSVC has no C99 support, and thus it cannot be used to - * compile liblzma. The liblzma API has to still be usable - * from MSVC, so we need to define the required standard - * integer types here. + * MSVC versions older than 2013 have no C99 support, and + * thus they cannot be used to compile liblzma. Using an + * existing liblzma.dll with old MSVC can work though(*), + * but we need to define the required standard integer + * types here in a MSVC-specific way. + * + * (*) If you do this, the existing liblzma.dll probably uses + * a different runtime library than your MSVC-built + * application. Mixing runtimes is generally bad, but + * in this case it should work as long as you avoid + * the few rarely-needed liblzma functions that allocate + * memory and expect the caller to free it using free(). */ -# if defined(_WIN32) && defined(_MSC_VER) +# if defined(_WIN32) && defined(_MSC_VER) && _MSC_VER < 1800 typedef unsigned __int8 uint8_t; typedef unsigned __int32 uint32_t; typedef unsigned __int64 uint64_t; @@ -211,7 +219,11 @@ */ #ifndef lzma_nothrow # if defined(__cplusplus) -# define lzma_nothrow throw() +# if __cplusplus >= 201103L +# define lzma_nothrow noexcept +# else +# define lzma_nothrow throw() +# endif # elif __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 3) # define lzma_nothrow __attribute__((__nothrow__)) # else @@ -286,7 +298,7 @@ extern "C" { #include "lzma/filter.h" #include "lzma/bcj.h" #include "lzma/delta.h" -#include "lzma/lzma.h" +#include "lzma/lzma12.h" /* Container formats */ #include "lzma/container.h" diff --git a/Utilities/cmliblzma/liblzma/api/lzma/base.h b/Utilities/cmliblzma/liblzma/api/lzma/base.h index 43dde8d..a6005ac 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/base.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/base.h @@ -240,12 +240,12 @@ typedef enum { /** * \brief The `action' argument for lzma_code() * - * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, or LZMA_FINISH, - * the same `action' must is used until lzma_code() returns LZMA_STREAM_END. - * Also, the amount of input (that is, strm->avail_in) must not be modified - * by the application until lzma_code() returns LZMA_STREAM_END. Changing the - * `action' or modifying the amount of input will make lzma_code() return - * LZMA_PROG_ERROR. + * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, LZMA_FULL_BARRIER, + * or LZMA_FINISH, the same `action' must is used until lzma_code() returns + * LZMA_STREAM_END. Also, the amount of input (that is, strm->avail_in) must + * not be modified by the application until lzma_code() returns + * LZMA_STREAM_END. Changing the `action' or modifying the amount of input + * will make lzma_code() return LZMA_PROG_ERROR. */ typedef enum { LZMA_RUN = 0, @@ -293,7 +293,7 @@ typedef enum { * * All the input data going to the current Block must have * been given to the encoder (the last bytes can still be - * pending in* next_in). Call lzma_code() with LZMA_FULL_FLUSH + * pending in *next_in). Call lzma_code() with LZMA_FULL_FLUSH * until it returns LZMA_STREAM_END. Then continue normally * with LZMA_RUN or finish the Stream with LZMA_FINISH. * @@ -302,6 +302,29 @@ typedef enum { * no unfinished Block, no empty Block is created. */ + LZMA_FULL_BARRIER = 4, + /**< + * \brief Finish encoding of the current Block + * + * This is like LZMA_FULL_FLUSH except that this doesn't + * necessarily wait until all the input has been made + * available via the output buffer. That is, lzma_code() + * might return LZMA_STREAM_END as soon as all the input + * has been consumed (avail_in == 0). + * + * LZMA_FULL_BARRIER is useful with a threaded encoder if + * one wants to split the .xz Stream into Blocks at specific + * offsets but doesn't care if the output isn't flushed + * immediately. Using LZMA_FULL_BARRIER allows keeping + * the threads busy while LZMA_FULL_FLUSH would make + * lzma_code() wait until all the threads have finished + * until more data could be passed to the encoder. + * + * With a lzma_stream initialized with the single-threaded + * lzma_stream_encoder() or lzma_easy_encoder(), + * LZMA_FULL_BARRIER is an alias for LZMA_FULL_FLUSH. + */ + LZMA_FINISH = 3 /**< * \brief Finish the coding operation @@ -332,11 +355,19 @@ typedef enum { * malloc() and free(). C++ users should note that the custom memory * handling functions must not throw exceptions. * - * liblzma doesn't make an internal copy of lzma_allocator. Thus, it is - * OK to change these function pointers in the middle of the coding - * process, but obviously it must be done carefully to make sure that the - * replacement `free' can deallocate memory allocated by the earlier - * `alloc' function(s). + * Single-threaded mode only: liblzma doesn't make an internal copy of + * lzma_allocator. Thus, it is OK to change these function pointers in + * the middle of the coding process, but obviously it must be done + * carefully to make sure that the replacement `free' can deallocate + * memory allocated by the earlier `alloc' function(s). + * + * Multithreaded mode: liblzma might internally store pointers to the + * lzma_allocator given via the lzma_stream structure. The application + * must not change the allocator pointer in lzma_stream or the contents + * of the pointed lzma_allocator structure until lzma_end() has been used + * to free the memory associated with that lzma_stream. The allocation + * functions might be called simultaneously from multiple threads, and + * thus they must be thread safe. */ typedef struct { /** @@ -448,7 +479,8 @@ typedef struct lzma_internal_s lzma_internal; * * Application may modify the values of total_in and total_out as it wants. * They are updated by liblzma to match the amount of data read and - * written, but aren't used for anything else. + * written but aren't used for anything else except as a possible return + * values from lzma_get_progress(). */ typedef struct { const uint8_t *next_in; /**< Pointer to the next input byte. */ @@ -464,8 +496,10 @@ typedef struct { * * In most cases this is NULL which makes liblzma use * the standard malloc() and free(). + * + * \note In 5.0.x this is not a const pointer. */ - lzma_allocator *allocator; + const lzma_allocator *allocator; /** Internal state is not visible to applications. */ lzma_internal *internal; @@ -547,6 +581,25 @@ extern LZMA_API(void) lzma_end(lzma_stream *strm) lzma_nothrow; /** + * \brief Get progress information + * + * In single-threaded mode, applications can get progress information from + * strm->total_in and strm->total_out. In multi-threaded mode this is less + * useful because a significant amount of both input and output data gets + * buffered internally by liblzma. This makes total_in and total_out give + * misleading information and also makes the progress indicator updates + * non-smooth. + * + * This function gives realistic progress information also in multi-threaded + * mode by taking into account the progress made by each thread. In + * single-threaded mode *progress_in and *progress_out are set to + * strm->total_in and strm->total_out, respectively. + */ +extern LZMA_API(void) lzma_get_progress(lzma_stream *strm, + uint64_t *progress_in, uint64_t *progress_out) lzma_nothrow; + + +/** * \brief Get the memory usage of decoder filter chain * * This function is currently supported only when *strm has been initialized @@ -591,11 +644,16 @@ extern LZMA_API(uint64_t) lzma_memlimit_get(const lzma_stream *strm) * This function is supported only when *strm has been initialized with * a function that takes a memlimit argument. * + * liblzma 5.2.3 and earlier has a bug where memlimit value of 0 causes + * this function to do nothing (leaving the limit unchanged) and still + * return LZMA_OK. Later versions treat 0 as if 1 had been specified (so + * lzma_memlimit_get() will return 1 even if you specify 0 here). + * * \return - LZMA_OK: New memory usage limit successfully set. * - LZMA_MEMLIMIT_ERROR: The new limit is too small. * The limit was not changed. * - LZMA_PROG_ERROR: Invalid arguments, e.g. *strm doesn't - * support memory usage limit or memlimit was zero. + * support memory usage limit. */ extern LZMA_API(lzma_ret) lzma_memlimit_set( lzma_stream *strm, uint64_t memlimit) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/block.h b/Utilities/cmliblzma/liblzma/api/lzma/block.h index e6710a7..7bdcfd7 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/block.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/block.h @@ -31,11 +31,16 @@ typedef struct { /** * \brief Block format version * - * To prevent API and ABI breakages if new features are needed in - * the Block field, a version number is used to indicate which - * fields in this structure are in use. For now, version must always - * be zero. With non-zero version, most Block related functions will - * return LZMA_OPTIONS_ERROR. + * To prevent API and ABI breakages when new features are needed, + * a version number is used to indicate which fields in this + * structure are in use: + * - liblzma >= 5.0.0: version = 0 is supported. + * - liblzma >= 5.1.4beta: Support for version = 1 was added, + * which adds the ignore_check field. + * + * If version is greater than one, most Block related functions + * will return LZMA_OPTIONS_ERROR (lzma_block_header_decode() works + * with any version value). * * Read by: * - All functions that take pointer to lzma_block as argument, @@ -233,7 +238,28 @@ typedef struct { lzma_reserved_enum reserved_enum2; lzma_reserved_enum reserved_enum3; lzma_reserved_enum reserved_enum4; - lzma_bool reserved_bool1; + + /** + * \brief A flag to Block decoder to not verify the Check field + * + * This field is supported by liblzma >= 5.1.4beta if .version >= 1. + * + * If this is set to true, the integrity check won't be calculated + * and verified. Unless you know what you are doing, you should + * leave this to false. (A reason to set this to true is when the + * file integrity is verified externally anyway and you want to + * speed up the decompression, which matters mostly when using + * SHA-256 as the integrity check.) + * + * If .version >= 1, read by: + * - lzma_block_decoder() + * - lzma_block_buffer_decode() + * + * Written by (.version is ignored): + * - lzma_block_header_decode() always sets this to false + */ + lzma_bool ignore_check; + lzma_bool reserved_bool2; lzma_bool reserved_bool3; lzma_bool reserved_bool4; @@ -310,10 +336,14 @@ extern LZMA_API(lzma_ret) lzma_block_header_encode( /** * \brief Decode Block Header * - * block->version should be set to the highest value supported by the - * application; currently the only possible version is zero. This function - * will set version to the lowest value that still supports all the features - * required by the Block Header. + * block->version should (usually) be set to the highest value supported + * by the application. If the application sets block->version to a value + * higher than supported by the current liblzma version, this function will + * downgrade block->version to the highest value supported by it. Thus one + * should check the value of block->version after calling this function if + * block->version was set to a non-zero value and the application doesn't + * otherwise know that the liblzma version being used is new enough to + * support the specified block->version. * * The size of the Block Header must have already been decoded with * lzma_block_header_size_decode() macro and stored to block->header_size. @@ -344,7 +374,7 @@ extern LZMA_API(lzma_ret) lzma_block_header_encode( * block->header_size is invalid or block->filters is NULL. */ extern LZMA_API(lzma_ret) lzma_block_header_decode(lzma_block *block, - lzma_allocator *allocator, const uint8_t *in) + const lzma_allocator *allocator, const uint8_t *in) lzma_nothrow lzma_attr_warn_unused_result; @@ -493,7 +523,25 @@ extern LZMA_API(size_t) lzma_block_buffer_bound(size_t uncompressed_size) * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_block_buffer_encode( - lzma_block *block, lzma_allocator *allocator, + lzma_block *block, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** + * \brief Single-call uncompressed .xz Block encoder + * + * This is like lzma_block_buffer_encode() except this doesn't try to + * compress the data and instead encodes the data using LZMA2 uncompressed + * chunks. The required output buffer size can be determined with + * lzma_block_buffer_bound(). + * + * Since the data won't be compressed, this function ignores block->filters. + * This function doesn't take lzma_allocator because this function doesn't + * allocate any memory from the heap. + */ +extern LZMA_API(lzma_ret) lzma_block_uncomp_encode(lzma_block *block, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -527,7 +575,7 @@ extern LZMA_API(lzma_ret) lzma_block_buffer_encode( * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_block_buffer_decode( - lzma_block *block, lzma_allocator *allocator, + lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/container.h b/Utilities/cmliblzma/liblzma/api/lzma/container.h index 7a9ffc6..9fbf4df 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/container.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/container.h @@ -61,6 +61,131 @@ /** + * \brief Multithreading options + */ +typedef struct { + /** + * \brief Flags + * + * Set this to zero if no flags are wanted. + * + * No flags are currently supported. + */ + uint32_t flags; + + /** + * \brief Number of worker threads to use + */ + uint32_t threads; + + /** + * \brief Maximum uncompressed size of a Block + * + * The encoder will start a new .xz Block every block_size bytes. + * Using LZMA_FULL_FLUSH or LZMA_FULL_BARRIER with lzma_code() + * the caller may tell liblzma to start a new Block earlier. + * + * With LZMA2, a recommended block size is 2-4 times the LZMA2 + * dictionary size. With very small dictionaries, it is recommended + * to use at least 1 MiB block size for good compression ratio, even + * if this is more than four times the dictionary size. Note that + * these are only recommendations for typical use cases; feel free + * to use other values. Just keep in mind that using a block size + * less than the LZMA2 dictionary size is waste of RAM. + * + * Set this to 0 to let liblzma choose the block size depending + * on the compression options. For LZMA2 it will be 3*dict_size + * or 1 MiB, whichever is more. + * + * For each thread, about 3 * block_size bytes of memory will be + * allocated. This may change in later liblzma versions. If so, + * the memory usage will probably be reduced, not increased. + */ + uint64_t block_size; + + /** + * \brief Timeout to allow lzma_code() to return early + * + * Multithreading can make liblzma to consume input and produce + * output in a very bursty way: it may first read a lot of input + * to fill internal buffers, then no input or output occurs for + * a while. + * + * In single-threaded mode, lzma_code() won't return until it has + * either consumed all the input or filled the output buffer. If + * this is done in multithreaded mode, it may cause a call + * lzma_code() to take even tens of seconds, which isn't acceptable + * in all applications. + * + * To avoid very long blocking times in lzma_code(), a timeout + * (in milliseconds) may be set here. If lzma_code() would block + * longer than this number of milliseconds, it will return with + * LZMA_OK. Reasonable values are 100 ms or more. The xz command + * line tool uses 300 ms. + * + * If long blocking times are fine for you, set timeout to a special + * value of 0, which will disable the timeout mechanism and will make + * lzma_code() block until all the input is consumed or the output + * buffer has been filled. + * + * \note Even with a timeout, lzma_code() might sometimes take + * somewhat long time to return. No timing guarantees + * are made. + */ + uint32_t timeout; + + /** + * \brief Compression preset (level and possible flags) + * + * The preset is set just like with lzma_easy_encoder(). + * The preset is ignored if filters below is non-NULL. + */ + uint32_t preset; + + /** + * \brief Filter chain (alternative to a preset) + * + * If this is NULL, the preset above is used. Otherwise the preset + * is ignored and the filter chain specified here is used. + */ + const lzma_filter *filters; + + /** + * \brief Integrity check type + * + * See check.h for available checks. The xz command line tool + * defaults to LZMA_CHECK_CRC64, which is a good choice if you + * are unsure. + */ + lzma_check check; + + /* + * Reserved space to allow possible future extensions without + * breaking the ABI. You should not touch these, because the names + * of these variables may change. These are and will never be used + * with the currently supported options, so it is safe to leave these + * uninitialized. + */ + lzma_reserved_enum reserved_enum1; + lzma_reserved_enum reserved_enum2; + lzma_reserved_enum reserved_enum3; + uint32_t reserved_int1; + uint32_t reserved_int2; + uint32_t reserved_int3; + uint32_t reserved_int4; + uint64_t reserved_int5; + uint64_t reserved_int6; + uint64_t reserved_int7; + uint64_t reserved_int8; + void *reserved_ptr1; + void *reserved_ptr2; + void *reserved_ptr3; + void *reserved_ptr4; + +} lzma_mt; + + +/** * \brief Calculate approximate memory usage of easy encoder * * This function is a wrapper for lzma_raw_encoder_memusage(). @@ -165,7 +290,8 @@ extern LZMA_API(lzma_ret) lzma_easy_encoder( */ extern LZMA_API(lzma_ret) lzma_easy_buffer_encode( uint32_t preset, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -191,6 +317,49 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder(lzma_stream *strm, /** + * \brief Calculate approximate memory usage of multithreaded .xz encoder + * + * Since doing the encoding in threaded mode doesn't affect the memory + * requirements of single-threaded decompressor, you can use + * lzma_easy_decoder_memusage(options->preset) or + * lzma_raw_decoder_memusage(options->filters) to calculate + * the decompressor memory requirements. + * + * \param options Compression options + * + * \return Number of bytes of memory required for encoding with the + * given options. If an error occurs, for example due to + * unsupported preset or filter chain, UINT64_MAX is returned. + */ +extern LZMA_API(uint64_t) lzma_stream_encoder_mt_memusage( + const lzma_mt *options) lzma_nothrow lzma_attr_pure; + + +/** + * \brief Initialize multithreaded .xz Stream encoder + * + * This provides the functionality of lzma_easy_encoder() and + * lzma_stream_encoder() as a single function for multithreaded use. + * + * The supported actions for lzma_code() are LZMA_RUN, LZMA_FULL_FLUSH, + * LZMA_FULL_BARRIER, and LZMA_FINISH. Support for LZMA_SYNC_FLUSH might be + * added in the future. + * + * \param strm Pointer to properly prepared lzma_stream + * \param options Pointer to multithreaded compression options + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_UNSUPPORTED_CHECK + * - LZMA_OPTIONS_ERROR + * - LZMA_PROG_ERROR + */ +extern LZMA_API(lzma_ret) lzma_stream_encoder_mt( + lzma_stream *strm, const lzma_mt *options) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** * \brief Initialize .lzma encoder (legacy file format) * * The .lzma format is sometimes called the LZMA_Alone format, which is the @@ -269,7 +438,8 @@ extern LZMA_API(size_t) lzma_stream_buffer_bound(size_t uncompressed_size) */ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( lzma_filter *filters, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -305,6 +475,30 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( /** + * This flag makes lzma_code() not calculate and verify the integrity check + * of the compressed data in .xz files. This means that invalid integrity + * check values won't be detected and LZMA_DATA_ERROR won't be returned in + * such cases. + * + * This flag only affects the checks of the compressed data itself; the CRC32 + * values in the .xz headers will still be verified normally. + * + * Don't use this flag unless you know what you are doing. Possible reasons + * to use this flag: + * + * - Trying to recover data from a corrupt .xz file. + * + * - Speeding up decompression, which matters mostly with SHA-256 + * or with files that have compressed extremely well. It's recommended + * to not use this flag for this purpose unless the file integrity is + * verified externally in some other way. + * + * Support for this flag was added in liblzma 5.1.4beta. + */ +#define LZMA_IGNORE_CHECK UINT32_C(0x10) + + +/** * This flag enables decoding of concatenated files with file formats that * allow concatenating compressed files as is. From the formats currently * supported by liblzma, only the .xz format allows concatenated files. @@ -326,7 +520,10 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( * * \param strm Pointer to properly prepared lzma_stream * \param memlimit Memory usage limit as bytes. Use UINT64_MAX - * to effectively disable the limiter. + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. * \param flags Bitwise-or of zero or more of the decoder flags: * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, * LZMA_TELL_ANY_CHECK, LZMA_CONCATENATED @@ -350,7 +547,10 @@ extern LZMA_API(lzma_ret) lzma_stream_decoder( * * \param strm Pointer to properly prepared lzma_stream * \param memlimit Memory usage limit as bytes. Use UINT64_MAX - * to effectively disable the limiter. + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. * \param flags Bitwise-or of flags, or zero for no flags. * * \return - LZMA_OK: Initialization was successful. @@ -366,9 +566,16 @@ extern LZMA_API(lzma_ret) lzma_auto_decoder( /** * \brief Initialize .lzma decoder (legacy file format) * + * \param strm Pointer to properly prepared lzma_stream + * \param memlimit Memory usage limit as bytes. Use UINT64_MAX + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. + * * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. - * There is no need to use LZMA_FINISH, but allowing it may simplify - * certain types of applications. + * There is no need to use LZMA_FINISH, but it's allowed because it may + * simplify certain types of applications. * * \return - LZMA_OK * - LZMA_MEM_ERROR @@ -418,7 +625,8 @@ extern LZMA_API(lzma_ret) lzma_alone_decoder( * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_buffer_decode( - uint64_t *memlimit, uint32_t flags, lzma_allocator *allocator, + uint64_t *memlimit, uint32_t flags, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/filter.h b/Utilities/cmliblzma/liblzma/api/lzma/filter.h index e0bc163..4e78752 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/filter.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/filter.h @@ -116,8 +116,9 @@ extern LZMA_API(lzma_bool) lzma_filter_decoder_is_supported(lzma_vli id) * is not NULL. * - LZMA_PROG_ERROR: src or dest is NULL. */ -extern LZMA_API(lzma_ret) lzma_filters_copy(const lzma_filter *src, - lzma_filter *dest, lzma_allocator *allocator) lzma_nothrow; +extern LZMA_API(lzma_ret) lzma_filters_copy( + const lzma_filter *src, lzma_filter *dest, + const lzma_allocator *allocator) lzma_nothrow; /** @@ -256,7 +257,7 @@ extern LZMA_API(lzma_ret) lzma_filters_update( * won't necessarily meet that bound.) */ extern LZMA_API(lzma_ret) lzma_raw_buffer_encode( - const lzma_filter *filters, lzma_allocator *allocator, + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -280,7 +281,7 @@ extern LZMA_API(lzma_ret) lzma_raw_buffer_encode( * which no data is written to is out[out_size]. */ extern LZMA_API(lzma_ret) lzma_raw_buffer_decode( - const lzma_filter *filters, lzma_allocator *allocator, + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -356,7 +357,7 @@ extern LZMA_API(lzma_ret) lzma_properties_encode( * - LZMA_MEM_ERROR */ extern LZMA_API(lzma_ret) lzma_properties_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) lzma_nothrow; @@ -419,6 +420,6 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filter, * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_filter_flags_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) lzma_nothrow lzma_attr_warn_unused_result; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/hardware.h b/Utilities/cmliblzma/liblzma/api/lzma/hardware.h index e7dd03c..5321d9a 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/hardware.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/hardware.h @@ -48,3 +48,17 @@ * of RAM on the specific operating system. */ extern LZMA_API(uint64_t) lzma_physmem(void) lzma_nothrow; + + +/** + * \brief Get the number of processor cores or threads + * + * This function may be useful when determining how many threads to use. + * If the hardware supports more than one thread per CPU core, the number + * of hardware threads is returned if that information is available. + * + * \brief On success, the number of available CPU threads or cores is + * returned. If this information isn't available or an error + * occurs, zero is returned. + */ +extern LZMA_API(uint32_t) lzma_cputhreads(void) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/index.h b/Utilities/cmliblzma/liblzma/api/lzma/index.h index 16bacc2..3dac6fb 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/index.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/index.h @@ -303,7 +303,7 @@ extern LZMA_API(uint64_t) lzma_index_memused(const lzma_index *i) * \return On success, a pointer to an empty initialized lzma_index is * returned. If allocation fails, NULL is returned. */ -extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator) +extern LZMA_API(lzma_index *) lzma_index_init(const lzma_allocator *allocator) lzma_nothrow; @@ -312,8 +312,8 @@ extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator) * * If i is NULL, this does nothing. */ -extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator) - lzma_nothrow; +extern LZMA_API(void) lzma_index_end( + lzma_index *i, const lzma_allocator *allocator) lzma_nothrow; /** @@ -341,7 +341,7 @@ extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator) * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_index_append( - lzma_index *i, lzma_allocator *allocator, + lzma_index *i, const lzma_allocator *allocator, lzma_vli unpadded_size, lzma_vli uncompressed_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -564,8 +564,8 @@ extern LZMA_API(lzma_bool) lzma_index_iter_locate( * - LZMA_MEM_ERROR * - LZMA_PROG_ERROR */ -extern LZMA_API(lzma_ret) lzma_index_cat( - lzma_index *dest, lzma_index *src, lzma_allocator *allocator) +extern LZMA_API(lzma_ret) lzma_index_cat(lzma_index *dest, lzma_index *src, + const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -575,7 +575,7 @@ extern LZMA_API(lzma_ret) lzma_index_cat( * \return A copy of the lzma_index, or NULL if memory allocation failed. */ extern LZMA_API(lzma_index *) lzma_index_dup( - const lzma_index *i, lzma_allocator *allocator) + const lzma_index *i, const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -586,8 +586,7 @@ extern LZMA_API(lzma_index *) lzma_index_dup( * \param i Pointer to lzma_index which should be encoded. * * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. - * It is enough to use only one of them (you can choose freely; use LZMA_RUN - * to support liblzma versions older than 5.0.0). + * It is enough to use only one of them (you can choose freely). * * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR @@ -610,16 +609,21 @@ extern LZMA_API(lzma_ret) lzma_index_encoder( * to a new lzma_index, which the application * has to later free with lzma_index_end(). * \param memlimit How much memory the resulting lzma_index is - * allowed to require. + * allowed to require. liblzma 5.2.3 and earlier + * don't allow 0 here and return LZMA_PROG_ERROR; + * later versions treat 0 as if 1 had been specified. * - * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. - * It is enough to use only one of them (you can choose freely; use LZMA_RUN - * to support liblzma versions older than 5.0.0). + * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * There is no need to use LZMA_FINISH, but it's allowed because it may + * simplify certain types of applications. * * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR - * - LZMA_MEMLIMIT_ERROR * - LZMA_PROG_ERROR + * + * liblzma 5.2.3 and older list also LZMA_MEMLIMIT_ERROR here + * but that error code has never been possible from this + * initialization function. */ extern LZMA_API(lzma_ret) lzma_index_decoder( lzma_stream *strm, lzma_index **i, uint64_t memlimit) @@ -677,6 +681,6 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i, * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_index_buffer_decode(lzma_index **i, - uint64_t *memlimit, lzma_allocator *allocator, + uint64_t *memlimit, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h b/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h index fa2e048..9287f1d 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h @@ -37,7 +37,7 @@ typedef struct lzma_index_hash_s lzma_index_hash; * pointer than the index_hash that was given as an argument. */ extern LZMA_API(lzma_index_hash *) lzma_index_hash_init( - lzma_index_hash *index_hash, lzma_allocator *allocator) + lzma_index_hash *index_hash, const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -45,7 +45,7 @@ extern LZMA_API(lzma_index_hash *) lzma_index_hash_init( * \brief Deallocate lzma_index_hash structure */ extern LZMA_API(void) lzma_index_hash_end( - lzma_index_hash *index_hash, lzma_allocator *allocator) + lzma_index_hash *index_hash, const lzma_allocator *allocator) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/lzma.h b/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h index 3f8e095..4e32fa3 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/lzma.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h @@ -1,5 +1,5 @@ /** - * \file lzma/lzma.h + * \file lzma/lzma12.h * \brief LZMA1 and LZMA2 filters */ diff --git a/Utilities/cmliblzma/liblzma/api/lzma/version.h b/Utilities/cmliblzma/liblzma/api/lzma/version.h index 09866b9..143c7de 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/version.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/version.h @@ -21,8 +21,8 @@ * Version number split into components */ #define LZMA_VERSION_MAJOR 5 -#define LZMA_VERSION_MINOR 0 -#define LZMA_VERSION_PATCH 8 +#define LZMA_VERSION_MINOR 2 +#define LZMA_VERSION_PATCH 4 #define LZMA_VERSION_STABILITY LZMA_VERSION_STABILITY_STABLE #ifndef LZMA_VERSION_COMMIT diff --git a/Utilities/cmliblzma/liblzma/check/check.c b/Utilities/cmliblzma/liblzma/check/check.c index 979b0a8..428ddae 100644 --- a/Utilities/cmliblzma/liblzma/check/check.c +++ b/Utilities/cmliblzma/liblzma/check/check.c @@ -16,6 +16,9 @@ extern LZMA_API(lzma_bool) lzma_check_is_supported(lzma_check type) { + if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) + return false; + static const lzma_bool available_checks[LZMA_CHECK_ID_MAX + 1] = { true, // LZMA_CHECK_NONE @@ -53,9 +56,6 @@ lzma_check_is_supported(lzma_check type) false, // Reserved }; - if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) - return false; - return available_checks[(unsigned int)(type)]; } @@ -63,6 +63,9 @@ lzma_check_is_supported(lzma_check type) extern LZMA_API(uint32_t) lzma_check_size(lzma_check type) { + if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) + return UINT32_MAX; + // See file-format.txt section 2.1.1.2. static const uint8_t check_sizes[LZMA_CHECK_ID_MAX + 1] = { 0, @@ -73,9 +76,6 @@ lzma_check_size(lzma_check type) 64, 64, 64 }; - if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) - return UINT32_MAX; - return check_sizes[(unsigned int)(type)]; } diff --git a/Utilities/cmliblzma/liblzma/check/check.h b/Utilities/cmliblzma/liblzma/check/check.h index e100d2b..3007d88 100644 --- a/Utilities/cmliblzma/liblzma/check/check.h +++ b/Utilities/cmliblzma/liblzma/check/check.h @@ -15,6 +15,53 @@ #include "common.h" +// If the function for external SHA-256 is missing, use the internal SHA-256 +// code. Due to how configure works, these defines can only get defined when +// both a usable header and a type have already been found. +#if !(defined(HAVE_CC_SHA256_INIT) \ + || defined(HAVE_SHA256_INIT) \ + || defined(HAVE_SHA256INIT)) +# define HAVE_INTERNAL_SHA256 1 +#endif + +#if defined(HAVE_INTERNAL_SHA256) +// Nothing +#elif defined(HAVE_COMMONCRYPTO_COMMONDIGEST_H) +# include <CommonCrypto/CommonDigest.h> +#elif defined(HAVE_SHA256_H) +# include <sys/types.h> +# include <sha256.h> +#elif defined(HAVE_SHA2_H) +# include <sys/types.h> +# include <sha2.h> +#endif + +#if defined(HAVE_INTERNAL_SHA256) +/// State for the internal SHA-256 implementation +typedef struct { + /// Internal state + uint32_t state[8]; + + /// Size of the message excluding padding + uint64_t size; +} lzma_sha256_state; +#elif defined(HAVE_CC_SHA256_CTX) +typedef CC_SHA256_CTX lzma_sha256_state; +#elif defined(HAVE_SHA256_CTX) +typedef SHA256_CTX lzma_sha256_state; +#elif defined(HAVE_SHA2_CTX) +typedef SHA2_CTX lzma_sha256_state; +#endif + +#if defined(HAVE_INTERNAL_SHA256) +// Nothing +#elif defined(HAVE_CC_SHA256_INIT) +# define LZMA_SHA256FUNC(x) CC_SHA256_ ## x +#elif defined(HAVE_SHA256_INIT) +# define LZMA_SHA256FUNC(x) SHA256_ ## x +#elif defined(HAVE_SHA256INIT) +# define LZMA_SHA256FUNC(x) SHA256 ## x +#endif // Index hashing needs the best possible hash function (preferably // a cryptographic hash) for maximum reliability. @@ -43,14 +90,7 @@ typedef struct { union { uint32_t crc32; uint64_t crc64; - - struct { - /// Internal state - uint32_t state[8]; - - /// Size of the message excluding padding - uint64_t size; - } sha256; + lzma_sha256_state sha256; } state; } lzma_check_state; @@ -82,6 +122,8 @@ extern void lzma_check_update(lzma_check_state *check, lzma_check type, extern void lzma_check_finish(lzma_check_state *check, lzma_check type); +#ifndef LZMA_SHA256FUNC + /// Prepare SHA-256 state for new input. extern void lzma_sha256_init(lzma_check_state *check); @@ -92,4 +134,39 @@ extern void lzma_sha256_update( /// Finish the SHA-256 calculation and store the result to check->buffer.u8. extern void lzma_sha256_finish(lzma_check_state *check); + +#else + +static inline void +lzma_sha256_init(lzma_check_state *check) +{ + LZMA_SHA256FUNC(Init)(&check->state.sha256); +} + + +static inline void +lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) +{ +#if defined(HAVE_CC_SHA256_INIT) && SIZE_MAX > UINT32_MAX + // Darwin's CC_SHA256_Update takes uint32_t as the buffer size, + // so use a loop to support size_t. + while (size > UINT32_MAX) { + LZMA_SHA256FUNC(Update)(&check->state.sha256, buf, UINT32_MAX); + buf += UINT32_MAX; + size -= UINT32_MAX; + } +#endif + + LZMA_SHA256FUNC(Update)(&check->state.sha256, buf, size); +} + + +static inline void +lzma_sha256_finish(lzma_check_state *check) +{ + LZMA_SHA256FUNC(Final)(check->buffer.u8, &check->state.sha256); +} + +#endif + #endif diff --git a/Utilities/cmliblzma/liblzma/check/crc32_fast.c b/Utilities/cmliblzma/liblzma/check/crc32_fast.c index c2c3cb7..3de0263 100644 --- a/Utilities/cmliblzma/liblzma/check/crc32_fast.c +++ b/Utilities/cmliblzma/liblzma/check/crc32_fast.c @@ -33,8 +33,6 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) #endif if (size > 8) { - const uint8_t * limit; - // Fix the alignment, if needed. The if statement above // ensures that this won't read past the end of buf[]. while ((uintptr_t)(buf) & 7) { @@ -43,7 +41,7 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) } // Calculate the position where to stop. - limit = buf + (size & ~(size_t)(7)); + const uint8_t *const limit = buf + (size & ~(size_t)(7)); // Calculate how many bytes must be calculated separately // before returning the result. @@ -51,8 +49,6 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) // Calculate the CRC32 using the slice-by-eight algorithm. while (buf < limit) { - uint32_t tmp; - crc ^= *(const uint32_t *)(buf); buf += 4; @@ -61,7 +57,7 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) ^ lzma_crc32_table[5][C(crc)] ^ lzma_crc32_table[4][D(crc)]; - tmp = *(const uint32_t *)(buf); + const uint32_t tmp = *(const uint32_t *)(buf); buf += 4; // At least with some compilers, it is critical for diff --git a/Utilities/cmliblzma/liblzma/check/crc64_fast.c b/Utilities/cmliblzma/liblzma/check/crc64_fast.c index 1436557..52af29e 100644 --- a/Utilities/cmliblzma/liblzma/check/crc64_fast.c +++ b/Utilities/cmliblzma/liblzma/check/crc64_fast.c @@ -36,14 +36,12 @@ lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) #endif if (size > 4) { - const uint8_t *limit; - while ((uintptr_t)(buf) & 3) { crc = lzma_crc64_table[0][*buf++ ^ A1(crc)] ^ S8(crc); --size; } - limit = buf + (size & ~(size_t)(3)); + const uint8_t *const limit = buf + (size & ~(size_t)(3)); size &= (size_t)(3); while (buf < limit) { diff --git a/Utilities/cmliblzma/liblzma/check/sha256.c b/Utilities/cmliblzma/liblzma/check/sha256.c index 3af6aa6..5eede5c 100644 --- a/Utilities/cmliblzma/liblzma/check/sha256.c +++ b/Utilities/cmliblzma/liblzma/check/sha256.c @@ -21,22 +21,22 @@ // /////////////////////////////////////////////////////////////////////////////// -// Avoid bogus warnings in transform(). -#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 2) || __GNUC__ > 4 -# pragma GCC diagnostic ignored "-Wuninitialized" -#endif - #include "check.h" -// At least on x86, GCC is able to optimize this to a rotate instruction. -#define rotr_32(num, amount) ((num) >> (amount) | (num) << (32 - (amount))) +// Rotate a uint32_t. GCC can optimize this to a rotate instruction +// at least on x86. +static inline uint32_t +rotr_32(uint32_t num, unsigned amount) +{ + return (num >> amount) | (num << (32 - amount)); +} -#define blk0(i) (W[i] = data[i]) +#define blk0(i) (W[i] = conv32be(data[i])) #define blk2(i) (W[i & 15] += s1(W[(i - 2) & 15]) + W[(i - 7) & 15] \ + s0(W[(i - 15) & 15])) #define Ch(x, y, z) (z ^ (x & (y ^ z))) -#define Maj(x, y, z) ((x & y) | (z & (x | y))) +#define Maj(x, y, z) ((x & (y ^ z)) + (y & z)) #define a(i) T[(0 - i) & 7] #define b(i) T[(1 - i) & 7] @@ -47,16 +47,17 @@ #define g(i) T[(6 - i) & 7] #define h(i) T[(7 - i) & 7] -#define R(i) \ - h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] \ - + (j ? blk2(i) : blk0(i)); \ +#define R(i, j, blk) \ + h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] + blk; \ d(i) += h(i); \ h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) +#define R0(i) R(i, 0, blk0(i)) +#define R2(i) R(i, j, blk2(i)) -#define S0(x) (rotr_32(x, 2) ^ rotr_32(x, 13) ^ rotr_32(x, 22)) -#define S1(x) (rotr_32(x, 6) ^ rotr_32(x, 11) ^ rotr_32(x, 25)) -#define s0(x) (rotr_32(x, 7) ^ rotr_32(x, 18) ^ (x >> 3)) -#define s1(x) (rotr_32(x, 17) ^ rotr_32(x, 19) ^ (x >> 10)) +#define S0(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 9), 11), 2) +#define S1(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 14), 5), 6) +#define s0(x) (rotr_32(x ^ rotr_32(x, 11), 7) ^ (x >> 3)) +#define s1(x) (rotr_32(x ^ rotr_32(x, 2), 17) ^ (x >> 10)) static const uint32_t SHA256_K[64] = { @@ -84,17 +85,22 @@ transform(uint32_t state[8], const uint32_t data[16]) { uint32_t W[16]; uint32_t T[8]; - unsigned int j; // Copy state[] to working vars. memcpy(T, state, sizeof(T)); - // 64 operations, partially loop unrolled - for (j = 0; j < 64; j += 16) { - R( 0); R( 1); R( 2); R( 3); - R( 4); R( 5); R( 6); R( 7); - R( 8); R( 9); R(10); R(11); - R(12); R(13); R(14); R(15); + // The first 16 operations unrolled + R0( 0); R0( 1); R0( 2); R0( 3); + R0( 4); R0( 5); R0( 6); R0( 7); + R0( 8); R0( 9); R0(10); R0(11); + R0(12); R0(13); R0(14); R0(15); + + // The remaining 48 operations partially unrolled + for (unsigned int j = 16; j < 64; j += 16) { + R2( 0); R2( 1); R2( 2); R2( 3); + R2( 4); R2( 5); R2( 6); R2( 7); + R2( 8); R2( 9); R2(10); R2(11); + R2(12); R2(13); R2(14); R2(15); } // Add the working vars back into state[]. @@ -112,19 +118,7 @@ transform(uint32_t state[8], const uint32_t data[16]) static void process(lzma_check_state *check) { -#ifdef WORDS_BIGENDIAN transform(check->state.sha256.state, check->buffer.u32); - -#else - uint32_t data[16]; - size_t i; - - for (i = 0; i < 16; ++i) - data[i] = bswap32(check->buffer.u32[i]); - - transform(check->state.sha256.state, data); -#endif - return; } @@ -174,8 +168,6 @@ lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) extern void lzma_sha256_finish(lzma_check_state *check) { - size_t i; - // Add padding as described in RFC 3174 (it describes SHA-1 but // the same padding style is used for SHA-256 too). size_t pos = check->state.sha256.size & 0x3F; @@ -197,7 +189,7 @@ lzma_sha256_finish(lzma_check_state *check) process(check); - for (i = 0; i < 8; ++i) + for (size_t i = 0; i < 8; ++i) check->buffer.u32[i] = conv32be(check->state.sha256.state[i]); return; diff --git a/Utilities/cmliblzma/liblzma/common/alone_decoder.c b/Utilities/cmliblzma/liblzma/common/alone_decoder.c index 5f5e564..77d0a9b 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/alone_decoder.c @@ -15,7 +15,7 @@ #include "lz_decoder.h" -struct lzma_coder_s { +typedef struct { lzma_next_coder next; enum { @@ -46,17 +46,19 @@ struct lzma_coder_s { /// Options decoded from the header needed to initialize /// the LZMA decoder lzma_options_lzma options; -}; +} lzma_alone_coder; static lzma_ret -alone_decode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +alone_decode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_alone_coder *coder = coder_ptr; + while (*out_pos < out_size && (coder->sequence == SEQ_CODE || *in_pos < in_size)) switch (coder->sequence) { @@ -126,17 +128,19 @@ alone_decode(lzma_coder *coder, // Fall through case SEQ_CODER_INIT: { - lzma_ret ret; + if (coder->memusage > coder->memlimit) + return LZMA_MEMLIMIT_ERROR; lzma_filter_info filters[2] = { - { 0, &lzma_lzma_decoder_init, &coder->options }, - { 0, NULL, NULL } + { + .init = &lzma_lzma_decoder_init, + .options = &coder->options, + }, { + .init = NULL, + } }; - if (coder->memusage > coder->memlimit) - return LZMA_MEMLIMIT_ERROR; - - ret = lzma_next_filter_init(&coder->next, + const lzma_ret ret = lzma_next_filter_init(&coder->next, allocator, filters); if (ret != LZMA_OK) return ret; @@ -164,8 +168,9 @@ alone_decode(lzma_coder *coder, static void -alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +alone_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_alone_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -173,9 +178,11 @@ alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -alone_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +alone_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_alone_coder *coder = coder_ptr; + *memusage = coder->memusage; *old_memlimit = coder->memlimit; @@ -191,34 +198,34 @@ alone_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, extern lzma_ret -lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_alone_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, bool picky) { lzma_next_coder_init(&lzma_alone_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; + lzma_alone_coder *coder = next->coder; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_alone_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &alone_decode; next->end = &alone_decoder_end; next->memconfig = &alone_decoder_memconfig; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } - next->coder->sequence = SEQ_PROPERTIES; - next->coder->picky = picky; - next->coder->pos = 0; - next->coder->options.dict_size = 0; - next->coder->options.preset_dict = NULL; - next->coder->options.preset_dict_size = 0; - next->coder->uncompressed_size = 0; - next->coder->memlimit = memlimit; - next->coder->memusage = LZMA_MEMUSAGE_BASE; + coder->sequence = SEQ_PROPERTIES; + coder->picky = picky; + coder->pos = 0; + coder->options.dict_size = 0; + coder->options.preset_dict = NULL; + coder->options.preset_dict_size = 0; + coder->uncompressed_size = 0; + coder->memlimit = my_max(1, memlimit); + coder->memusage = LZMA_MEMUSAGE_BASE; return LZMA_OK; } @@ -227,7 +234,7 @@ lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit) { - lzma_next_strm_init2(lzma_alone_decoder_init, strm, memlimit, false); + lzma_next_strm_init(lzma_alone_decoder_init, strm, memlimit, false); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/alone_decoder.h b/Utilities/cmliblzma/liblzma/common/alone_decoder.h index f666fc3..dfa031a 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/alone_decoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_alone_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, bool picky); #endif diff --git a/Utilities/cmliblzma/liblzma/common/alone_encoder.c b/Utilities/cmliblzma/liblzma/common/alone_encoder.c index 4207b4a..4853cfd 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/alone_encoder.c @@ -17,7 +17,7 @@ #define ALONE_HEADER_SIZE (1 + 4 + 8) -struct lzma_coder_s { +typedef struct { lzma_next_coder next; enum { @@ -27,17 +27,19 @@ struct lzma_coder_s { size_t header_pos; uint8_t header[ALONE_HEADER_SIZE]; -}; +} lzma_alone_coder; static lzma_ret -alone_encode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +alone_encode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_alone_coder *coder = coder_ptr; + while (*out_pos < out_size) switch (coder->sequence) { case SEQ_HEADER: @@ -65,8 +67,9 @@ alone_encode(lzma_coder *coder, static void -alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +alone_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_alone_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -75,36 +78,31 @@ alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) // At least for now, this is not used by any internal function. static lzma_ret -alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_options_lzma *options) { - uint32_t d; - - // Initialize the LZMA encoder. - const lzma_filter_info filters[2] = { - { 0, &lzma_lzma_encoder_init, (void *)(options) }, - { 0, NULL, NULL } - }; - lzma_next_coder_init(&alone_encoder_init, next, allocator); - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_alone_coder *coder = next->coder; + + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_alone_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &alone_encode; next->end = &alone_encoder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_HEADER; - next->coder->header_pos = 0; + coder->sequence = SEQ_HEADER; + coder->header_pos = 0; // Encode the header: // - Properties (1 byte) - if (lzma_lzma_lclppb_encode(options, next->coder->header)) + if (lzma_lzma_lclppb_encode(options, coder->header)) return LZMA_OPTIONS_ERROR; // - Dictionary size (4 bytes) @@ -115,7 +113,7 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // one is the next unless it is UINT32_MAX. While the header would // allow any 32-bit integer, we do this to keep the decoder of liblzma // accepting the resulting files. - d = options->dict_size - 1; + uint32_t d = options->dict_size - 1; d |= d >> 2; d |= d >> 3; d |= d >> 4; @@ -124,18 +122,28 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, if (d != UINT32_MAX) ++d; - unaligned_write32le(next->coder->header + 1, d); + unaligned_write32le(coder->header + 1, d); // - Uncompressed size (always unknown and using EOPM) - memset(next->coder->header + 1 + 4, 0xFF, 8); + memset(coder->header + 1 + 4, 0xFF, 8); + + // Initialize the LZMA encoder. + const lzma_filter_info filters[2] = { + { + .init = &lzma_lzma_encoder_init, + .options = (void *)(options), + }, { + .init = NULL, + } + }; - return lzma_next_filter_init(&next->coder->next, allocator, filters); + return lzma_next_filter_init(&coder->next, allocator, filters); } /* extern lzma_ret -lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_options_alone *options) { lzma_next_coder_init(&alone_encoder_init, next, allocator, options); @@ -146,7 +154,7 @@ lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_alone_encoder(lzma_stream *strm, const lzma_options_lzma *options) { - lzma_next_strm_init1(alone_encoder_init, strm, options); + lzma_next_strm_init(alone_encoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/auto_decoder.c b/Utilities/cmliblzma/liblzma/common/auto_decoder.c index 24cf489..6895c7c 100644 --- a/Utilities/cmliblzma/liblzma/common/auto_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/auto_decoder.c @@ -14,7 +14,7 @@ #include "alone_decoder.h" -struct lzma_coder_s { +typedef struct { /// Stream decoder or LZMA_Alone decoder lzma_next_coder next; @@ -26,15 +26,17 @@ struct lzma_coder_s { SEQ_CODE, SEQ_FINISH, } sequence; -}; +} lzma_auto_coder; static lzma_ret -auto_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +auto_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_auto_coder *coder = coder_ptr; + switch (coder->sequence) { case SEQ_INIT: if (*in_pos >= in_size) @@ -100,8 +102,9 @@ auto_decode(lzma_coder *coder, lzma_allocator *allocator, static void -auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +auto_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_auto_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -109,8 +112,10 @@ auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_check -auto_decoder_get_check(const lzma_coder *coder) +auto_decoder_get_check(const void *coder_ptr) { + const lzma_auto_coder *coder = coder_ptr; + // It is LZMA_Alone if get_check is NULL. return coder->next.get_check == NULL ? LZMA_CHECK_NONE : coder->next.get_check(coder->next.coder); @@ -118,9 +123,11 @@ auto_decoder_get_check(const lzma_coder *coder) static lzma_ret -auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +auto_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_auto_coder *coder = coder_ptr; + lzma_ret ret; if (coder->next.memconfig != NULL) { @@ -132,7 +139,10 @@ auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, // the current memory usage. *memusage = LZMA_MEMUSAGE_BASE; *old_memlimit = coder->memlimit; + ret = LZMA_OK; + if (new_memlimit != 0 && new_memlimit < *memusage) + ret = LZMA_MEMLIMIT_ERROR; } if (ret == LZMA_OK && new_memlimit != 0) @@ -143,32 +153,31 @@ auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, static lzma_ret -auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +auto_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, uint32_t flags) { lzma_next_coder_init(&auto_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; - if (flags & ~LZMA_SUPPORTED_FLAGS) return LZMA_OPTIONS_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_auto_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_auto_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &auto_decode; next->end = &auto_decoder_end; next->get_check = &auto_decoder_get_check; next->memconfig = &auto_decoder_memconfig; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } - next->coder->memlimit = memlimit; - next->coder->flags = flags; - next->coder->sequence = SEQ_INIT; + coder->memlimit = my_max(1, memlimit); + coder->flags = flags; + coder->sequence = SEQ_INIT; return LZMA_OK; } @@ -177,7 +186,7 @@ auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_auto_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init2(auto_decoder_init, strm, memlimit, flags); + lzma_next_strm_init(auto_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c index b4bd388..b0ded90 100644 --- a/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c @@ -14,13 +14,10 @@ extern LZMA_API(lzma_ret) -lzma_block_buffer_decode(lzma_block *block, lzma_allocator *allocator, +lzma_block_buffer_decode(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder block_decoder; - lzma_ret ret; - if (in_pos == NULL || (in == NULL && *in_pos != in_size) || *in_pos > in_size || out_pos == NULL || (out == NULL && *out_pos != out_size) @@ -28,8 +25,9 @@ lzma_block_buffer_decode(lzma_block *block, lzma_allocator *allocator, return LZMA_PROG_ERROR; // Initialize the Block decoder. - block_decoder = LZMA_NEXT_CODER_INIT; - ret = lzma_block_decoder_init(&block_decoder, allocator, block); + lzma_next_coder block_decoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_block_decoder_init( + &block_decoder, allocator, block); if (ret == LZMA_OK) { // Save the positions so that we can restore them in case diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c index 136f7f5..39e263a 100644 --- a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c @@ -10,6 +10,7 @@ // /////////////////////////////////////////////////////////////////////////////// +#include "block_buffer_encoder.h" #include "block_encoder.h" #include "filter_encoder.h" #include "lzma2_encoder.h" @@ -28,11 +29,9 @@ + LZMA_CHECK_SIZE_MAX + 3) & ~3) -static lzma_vli -lzma2_bound(lzma_vli uncompressed_size) +static uint64_t +lzma2_bound(uint64_t uncompressed_size) { - lzma_vli overhead; - // Prevent integer overflow in overhead calculation. if (uncompressed_size > COMPRESSED_SIZE_MAX) return 0; @@ -41,7 +40,7 @@ lzma2_bound(lzma_vli uncompressed_size) // uncompressed_size up to the next multiple of LZMA2_CHUNK_MAX, // multiply by the size of per-chunk header, and add one byte for // the end marker. - overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1) + const uint64_t overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1) / LZMA2_CHUNK_MAX) * LZMA2_HEADER_UNCOMPRESSED + 1; @@ -53,30 +52,36 @@ lzma2_bound(lzma_vli uncompressed_size) } -extern LZMA_API(size_t) -lzma_block_buffer_bound(size_t uncompressed_size) +extern uint64_t +lzma_block_buffer_bound64(uint64_t uncompressed_size) { - // For now, if the data doesn't compress, we always use uncompressed - // chunks of LZMA2. In future we may use Subblock filter too, but - // but for simplicity we probably will still use the same bound - // calculation even though Subblock filter would have slightly less - // overhead. - lzma_vli lzma2_size = lzma2_bound(uncompressed_size); + // If the data doesn't compress, we always use uncompressed + // LZMA2 chunks. + uint64_t lzma2_size = lzma2_bound(uncompressed_size); if (lzma2_size == 0) return 0; // Take Block Padding into account. - lzma2_size = (lzma2_size + 3) & ~LZMA_VLI_C(3); + lzma2_size = (lzma2_size + 3) & ~UINT64_C(3); -#if SIZE_MAX < LZMA_VLI_MAX - // Catch the possible integer overflow on 32-bit systems. There's no - // overflow on 64-bit systems, because lzma2_bound() already takes + // No risk of integer overflow because lzma2_bound() already takes // into account the size of the headers in the Block. - if (SIZE_MAX - HEADERS_BOUND < lzma2_size) + return HEADERS_BOUND + lzma2_size; +} + + +extern LZMA_API(size_t) +lzma_block_buffer_bound(size_t uncompressed_size) +{ + uint64_t ret = lzma_block_buffer_bound64(uncompressed_size); + +#if SIZE_MAX < UINT64_MAX + // Catch the possible integer overflow on 32-bit systems. + if (ret > SIZE_MAX) return 0; #endif - return HEADERS_BOUND + lzma2_size; + return ret; } @@ -84,17 +89,12 @@ static lzma_ret block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - size_t in_pos = 0; - uint8_t control = 0x01; // Dictionary reset - lzma_filter *filters_orig; - - // TODO: Figure out if the last filter is LZMA2 or Subblock and use - // that filter to encode the uncompressed chunks. - // Use LZMA2 uncompressed chunks. We wouldn't need a dictionary at // all, but LZMA2 always requires a dictionary, so use the minimum // value to minimize memory usage of the decoder. - lzma_options_lzma lzma2 = { LZMA_DICT_SIZE_MIN }; + lzma_options_lzma lzma2 = { + .dict_size = LZMA_DICT_SIZE_MIN, + }; lzma_filter filters[2]; filters[0].id = LZMA_FILTER_LZMA2; @@ -103,7 +103,7 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, // Set the above filter options to *block temporarily so that we can // encode the Block Header. - filters_orig = block->filters; + lzma_filter *filters_orig = block->filters; block->filters = filters; if (lzma_block_header_size(block) != LZMA_OK) { @@ -132,17 +132,18 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, *out_pos += block->header_size; // Encode the data using LZMA2 uncompressed chunks. + size_t in_pos = 0; + uint8_t control = 0x01; // Dictionary reset while (in_pos < in_size) { - size_t copy_size; - // Control byte: Indicate uncompressed chunk, of which // the first resets the dictionary. out[(*out_pos)++] = control; control = 0x02; // No dictionary reset // Size of the uncompressed chunk - copy_size = my_min(in_size - in_pos, LZMA2_CHUNK_MAX); + const size_t copy_size + = my_min(in_size - in_pos, LZMA2_CHUNK_MAX); out[(*out_pos)++] = (copy_size - 1) >> 8; out[(*out_pos)++] = (copy_size - 1) & 0xFF; @@ -163,27 +164,18 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, static lzma_ret -block_encode_normal(lzma_block *block, lzma_allocator *allocator, +block_encode_normal(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - size_t out_start; - lzma_next_coder raw_encoder = LZMA_NEXT_CODER_INIT; - lzma_ret ret; - // Find out the size of the Block Header. - block->compressed_size = lzma2_bound(in_size); - if (block->compressed_size == 0) - return LZMA_DATA_ERROR; - - block->uncompressed_size = in_size; return_if_error(lzma_block_header_size(block)); // Reserve space for the Block Header and skip it for now. if (out_size - *out_pos <= block->header_size) return LZMA_BUF_ERROR; - out_start = *out_pos; + const size_t out_start = *out_pos; *out_pos += block->header_size; // Limit out_size so that we stop encoding if the output would grow @@ -193,7 +185,8 @@ block_encode_normal(lzma_block *block, lzma_allocator *allocator, // TODO: In many common cases this could be optimized to use // significantly less memory. - ret = lzma_raw_encoder_init( + lzma_next_coder raw_encoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_raw_encoder_init( &raw_encoder, allocator, block->filters); if (ret == LZMA_OK) { @@ -227,15 +220,12 @@ block_encode_normal(lzma_block *block, lzma_allocator *allocator, } -extern LZMA_API(lzma_ret) -lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, +static lzma_ret +block_buffer_encode(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, - uint8_t *out, size_t *out_pos, size_t out_size) + uint8_t *out, size_t *out_pos, size_t out_size, + bool try_to_compress) { - size_t check_size; - lzma_ret ret; - size_t i; - // Validate the arguments. if (block == NULL || (in == NULL && in_size != 0) || out == NULL || out_pos == NULL || *out_pos > out_size) @@ -243,11 +233,11 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, // The contents of the structure may depend on the version so // check the version before validating the contents of *block. - if (block->version != 0) + if (block->version > 1) return LZMA_OPTIONS_ERROR; if ((unsigned int)(block->check) > LZMA_CHECK_ID_MAX - || block->filters == NULL) + || (try_to_compress && block->filters == NULL)) return LZMA_PROG_ERROR; if (!lzma_check_is_supported(block->check)) @@ -259,7 +249,7 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, out_size -= (out_size - *out_pos) & 3; // Get the size of the Check field. - check_size = lzma_check_size(block->check); + const size_t check_size = lzma_check_size(block->check); assert(check_size != UINT32_MAX); // Reserve space for the Check field. @@ -268,9 +258,19 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, out_size -= check_size; + // Initialize block->uncompressed_size and calculate the worst-case + // value for block->compressed_size. + block->uncompressed_size = in_size; + block->compressed_size = lzma2_bound(in_size); + if (block->compressed_size == 0) + return LZMA_DATA_ERROR; + // Do the actual compression. - ret = block_encode_normal(block, allocator, - in, in_size, out, out_pos, out_size); + lzma_ret ret = LZMA_BUF_ERROR; + if (try_to_compress) + ret = block_encode_normal(block, allocator, + in, in_size, out, out_pos, out_size); + if (ret != LZMA_OK) { // If the error was something else than output buffer // becoming full, return the error now. @@ -291,7 +291,7 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, // Block Padding. No buffer overflow here, because we already adjusted // out_size so that (out_size - out_start) is a multiple of four. // Thus, if the buffer is full, the loop body can never run. - for (i = (size_t)(block->compressed_size); i & 3; ++i) { + for (size_t i = (size_t)(block->compressed_size); i & 3; ++i) { assert(*out_pos < out_size); out[(*out_pos)++] = 0x00; } @@ -313,3 +313,25 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, return LZMA_OK; } + + +extern LZMA_API(lzma_ret) +lzma_block_buffer_encode(lzma_block *block, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + return block_buffer_encode(block, allocator, + in, in_size, out, out_pos, out_size, true); +} + + +extern LZMA_API(lzma_ret) +lzma_block_uncomp_encode(lzma_block *block, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + // It won't allocate any memory from heap so no need + // for lzma_allocator. + return block_buffer_encode(block, NULL, + in, in_size, out, out_pos, out_size, false); +} diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h new file mode 100644 index 0000000..653207f --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h @@ -0,0 +1,24 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_buffer_encoder.h +/// \brief Single-call .xz Block encoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BLOCK_BUFFER_ENCODER_H +#define LZMA_BLOCK_BUFFER_ENCODER_H + +#include "common.h" + + +/// uint64_t version of lzma_block_buffer_bound(). It is used by +/// stream_encoder_mt.c. Probably the original lzma_block_buffer_bound() +/// should have been 64-bit, but fixing it would break the ABI. +extern uint64_t lzma_block_buffer_bound64(uint64_t uncompressed_size); + +#endif diff --git a/Utilities/cmliblzma/liblzma/common/block_decoder.c b/Utilities/cmliblzma/liblzma/common/block_decoder.c index 35996e7..075bd27 100644 --- a/Utilities/cmliblzma/liblzma/common/block_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_decoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_CODE, SEQ_PADDING, @@ -45,7 +45,10 @@ struct lzma_coder_s { /// Check of the uncompressed data lzma_check_state check; -}; + + /// True if the integrity check won't be calculated and verified. + bool ignore_check; +} lzma_block_coder; static inline bool @@ -71,11 +74,13 @@ is_size_valid(lzma_vli size, lzma_vli reference) static lzma_ret -block_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +block_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_block_coder *coder = coder_ptr; + switch (coder->sequence) { case SEQ_CODE: { const size_t in_start = *in_pos; @@ -97,8 +102,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, coder->block->uncompressed_size)) return LZMA_DATA_ERROR; - lzma_check_update(&coder->check, coder->block->check, - out + out_start, out_used); + if (!coder->ignore_check) + lzma_check_update(&coder->check, coder->block->check, + out + out_start, out_used); if (ret != LZMA_STREAM_END) return ret; @@ -140,7 +146,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, if (coder->block->check == LZMA_CHECK_NONE) return LZMA_STREAM_END; - lzma_check_finish(&coder->check, coder->block->check); + if (!coder->ignore_check) + lzma_check_finish(&coder->check, coder->block->check); + coder->sequence = SEQ_CHECK; // Fall through @@ -155,7 +163,8 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, // Validate the Check only if we support it. // coder->check.buffer may be uninitialized // when the Check ID is not supported. - if (lzma_check_is_supported(coder->block->check) + if (!coder->ignore_check + && lzma_check_is_supported(coder->block->check) && memcmp(coder->block->raw_check, coder->check.buffer.u8, check_size) != 0) @@ -170,8 +179,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, static void -block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +block_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_block_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -179,7 +189,7 @@ block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_block_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_block *block) { lzma_next_coder_init(&lzma_block_decoder_init, next, allocator); @@ -191,27 +201,29 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, || !lzma_vli_is_valid(block->uncompressed_size)) return LZMA_PROG_ERROR; - // Allocate and initialize *next->coder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + // Allocate *next->coder if needed. + lzma_block_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_block_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &block_decode; next->end = &block_decoder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_CODE; - next->coder->block = block; - next->coder->compressed_size = 0; - next->coder->uncompressed_size = 0; + coder->sequence = SEQ_CODE; + coder->block = block; + coder->compressed_size = 0; + coder->uncompressed_size = 0; // If Compressed Size is not known, we calculate the maximum allowed // value so that encoded size of the Block (including Block Padding) // is still a valid VLI and a multiple of four. - next->coder->compressed_limit + coder->compressed_limit = block->compressed_size == LZMA_VLI_UNKNOWN ? (LZMA_VLI_MAX & ~LZMA_VLI_C(3)) - block->header_size @@ -221,11 +233,14 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, // Initialize the check. It's caller's problem if the Check ID is not // supported, and the Block decoder cannot verify the Check field. // Caller can test lzma_check_is_supported(block->check). - next->coder->check_pos = 0; - lzma_check_init(&next->coder->check, block->check); + coder->check_pos = 0; + lzma_check_init(&coder->check, block->check); + + coder->ignore_check = block->version >= 1 + ? block->ignore_check : false; // Initialize the filter chain. - return lzma_raw_decoder_init(&next->coder->next, allocator, + return lzma_raw_decoder_init(&coder->next, allocator, block->filters); } @@ -233,7 +248,7 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_block_decoder(lzma_stream *strm, lzma_block *block) { - lzma_next_strm_init1(lzma_block_decoder_init, strm, block); + lzma_next_strm_init(lzma_block_decoder_init, strm, block); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_decoder.h b/Utilities/cmliblzma/liblzma/common/block_decoder.h index 7da9df6..718c5ce 100644 --- a/Utilities/cmliblzma/liblzma/common/block_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/block_decoder.h @@ -17,6 +17,6 @@ extern lzma_ret lzma_block_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_block *block); + const lzma_allocator *allocator, lzma_block *block); #endif diff --git a/Utilities/cmliblzma/liblzma/common/block_encoder.c b/Utilities/cmliblzma/liblzma/common/block_encoder.c index ed74827..168846a 100644 --- a/Utilities/cmliblzma/liblzma/common/block_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_encoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { /// The filters in the chain; initialized with lzma_raw_decoder_init(). lzma_next_coder next; @@ -41,15 +41,17 @@ struct lzma_coder_s { /// Check of the uncompressed data lzma_check_state check; -}; +} lzma_block_coder; static lzma_ret -block_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +block_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_block_coder *coder = coder_ptr; + // Check that our amount of input stays in proper limits. if (LZMA_VLI_MAX - coder->uncompressed_size < in_size - *in_pos) return LZMA_DATA_ERROR; @@ -134,8 +136,9 @@ block_encode(lzma_coder *coder, lzma_allocator *allocator, static void -block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +block_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_block_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -143,10 +146,12 @@ block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -block_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +block_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_block_coder *coder = coder_ptr; + if (coder->sequence != SEQ_CODE) return LZMA_PROG_ERROR; @@ -156,7 +161,7 @@ block_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_block_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_block *block) { lzma_next_coder_init(&lzma_block_encoder_init, next, allocator); @@ -166,7 +171,7 @@ lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // The contents of the structure may depend on the version so // check the version first. - if (block->version != 0) + if (block->version > 1) return LZMA_OPTIONS_ERROR; // If the Check ID is not supported, we cannot calculate the check and @@ -178,37 +183,38 @@ lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_UNSUPPORTED_CHECK; // Allocate and initialize *next->coder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_block_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_block_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &block_encode; next->end = &block_encoder_end; next->update = &block_encoder_update; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_CODE; - next->coder->block = block; - next->coder->compressed_size = 0; - next->coder->uncompressed_size = 0; - next->coder->pos = 0; + coder->sequence = SEQ_CODE; + coder->block = block; + coder->compressed_size = 0; + coder->uncompressed_size = 0; + coder->pos = 0; // Initialize the check - lzma_check_init(&next->coder->check, block->check); + lzma_check_init(&coder->check, block->check); // Initialize the requested filters. - return lzma_raw_encoder_init(&next->coder->next, allocator, - block->filters); + return lzma_raw_encoder_init(&coder->next, allocator, block->filters); } extern LZMA_API(lzma_ret) lzma_block_encoder(lzma_stream *strm, lzma_block *block) { - lzma_next_strm_init1(lzma_block_encoder_init, strm, block); + lzma_next_strm_init(lzma_block_encoder_init, strm, block); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_encoder.h b/Utilities/cmliblzma/liblzma/common/block_encoder.h index b9eff0b..bd97c18 100644 --- a/Utilities/cmliblzma/liblzma/common/block_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/block_encoder.h @@ -42,6 +42,6 @@ extern lzma_ret lzma_block_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_block *block); + const lzma_allocator *allocator, lzma_block *block); #endif diff --git a/Utilities/cmliblzma/liblzma/common/block_header_decoder.c b/Utilities/cmliblzma/liblzma/common/block_header_decoder.c index f6e470e..1dd982f 100644 --- a/Utilities/cmliblzma/liblzma/common/block_header_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_header_decoder.c @@ -15,14 +15,12 @@ static void -free_properties(lzma_block *block, lzma_allocator *allocator) +free_properties(lzma_block *block, const lzma_allocator *allocator) { - size_t i; - // Free allocated filter options. The last array member is not // touched after the initialization in the beginning of // lzma_block_header_decode(), so we don't need to touch that here. - for (i = 0; i < LZMA_FILTERS_MAX; ++i) { + for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) { lzma_free(block->filters[i].options, allocator); block->filters[i].id = LZMA_VLI_UNKNOWN; block->filters[i].options = NULL; @@ -34,15 +32,8 @@ free_properties(lzma_block *block, lzma_allocator *allocator) extern LZMA_API(lzma_ret) lzma_block_header_decode(lzma_block *block, - lzma_allocator *allocator, const uint8_t *in) + const lzma_allocator *allocator, const uint8_t *in) { - const size_t filter_count = (in[1] & 3) + 1; - size_t in_size; - size_t i; - - // Start after the Block Header Size and Block Flags fields. - size_t in_pos = 2; - // NOTE: We consider the header to be corrupt not only when the // CRC32 doesn't match, but also when variable-length integers // are invalid or over 63 bits, or if the header is too small @@ -50,13 +41,21 @@ lzma_block_header_decode(lzma_block *block, // Initialize the filter options array. This way the caller can // safely free() the options even if an error occurs in this function. - for (i = 0; i <= LZMA_FILTERS_MAX; ++i) { + for (size_t i = 0; i <= LZMA_FILTERS_MAX; ++i) { block->filters[i].id = LZMA_VLI_UNKNOWN; block->filters[i].options = NULL; } - // Always zero for now. - block->version = 0; + // Versions 0 and 1 are supported. If a newer version was specified, + // we need to downgrade it. + if (block->version > 1) + block->version = 1; + + // This isn't a Block Header option, but since the decompressor will + // read it if version >= 1, it's better to initialize it here than + // to expect the caller to do it since in almost all cases this + // should be false. + block->ignore_check = false; // Validate Block Header Size and Check type. The caller must have // already set these, so it is a programming error if this test fails. @@ -65,7 +64,7 @@ lzma_block_header_decode(lzma_block *block, return LZMA_PROG_ERROR; // Exclude the CRC32 field. - in_size = block->header_size - 4; + const size_t in_size = block->header_size - 4; // Verify CRC32 if (lzma_crc32(in, in_size, 0) != unaligned_read32le(in + in_size)) @@ -75,6 +74,9 @@ lzma_block_header_decode(lzma_block *block, if (in[1] & 0x3C) return LZMA_OPTIONS_ERROR; + // Start after the Block Header Size and Block Flags fields. + size_t in_pos = 2; + // Compressed Size if (in[1] & 0x40) { return_if_error(lzma_vli_decode(&block->compressed_size, @@ -96,7 +98,8 @@ lzma_block_header_decode(lzma_block *block, block->uncompressed_size = LZMA_VLI_UNKNOWN; // Filter Flags - for (i = 0; i < filter_count; ++i) { + const size_t filter_count = (in[1] & 3) + 1; + for (size_t i = 0; i < filter_count; ++i) { const lzma_ret ret = lzma_filter_flags_decode( &block->filters[i], allocator, in, &in_pos, in_size); diff --git a/Utilities/cmliblzma/liblzma/common/block_header_encoder.c b/Utilities/cmliblzma/liblzma/common/block_header_encoder.c index 650295c..5c5f542 100644 --- a/Utilities/cmliblzma/liblzma/common/block_header_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_header_encoder.c @@ -17,14 +17,12 @@ extern LZMA_API(lzma_ret) lzma_block_header_size(lzma_block *block) { - size_t i; + if (block->version > 1) + return LZMA_OPTIONS_ERROR; // Block Header Size + Block Flags + CRC32. uint32_t size = 1 + 1 + 4; - if (block->version != 0) - return LZMA_OPTIONS_ERROR; - // Compressed Size if (block->compressed_size != LZMA_VLI_UNKNOWN) { const uint32_t add = lzma_vli_size(block->compressed_size); @@ -47,13 +45,12 @@ lzma_block_header_size(lzma_block *block) if (block->filters == NULL || block->filters[0].id == LZMA_VLI_UNKNOWN) return LZMA_PROG_ERROR; - for (i = 0; block->filters[i].id != LZMA_VLI_UNKNOWN; ++i) { - uint32_t add; - + for (size_t i = 0; block->filters[i].id != LZMA_VLI_UNKNOWN; ++i) { // Don't allow too many filters. if (i == LZMA_FILTERS_MAX) return LZMA_PROG_ERROR; + uint32_t add; return_if_error(lzma_filter_flags_size(&add, block->filters + i)); @@ -76,23 +73,20 @@ lzma_block_header_size(lzma_block *block) extern LZMA_API(lzma_ret) lzma_block_header_encode(const lzma_block *block, uint8_t *out) { - size_t out_size; - size_t out_pos = 2; - size_t filter_count = 0; - // Validate everything but filters. if (lzma_block_unpadded_size(block) == 0 || !lzma_vli_is_valid(block->uncompressed_size)) return LZMA_PROG_ERROR; // Indicate the size of the buffer _excluding_ the CRC32 field. - out_size = block->header_size - 4; + const size_t out_size = block->header_size - 4; // Store the Block Header Size. out[0] = out_size / 4; // We write Block Flags in pieces. out[1] = 0x00; + size_t out_pos = 2; // Compressed Size if (block->compressed_size != LZMA_VLI_UNKNOWN) { @@ -114,6 +108,7 @@ lzma_block_header_encode(const lzma_block *block, uint8_t *out) if (block->filters == NULL || block->filters[0].id == LZMA_VLI_UNKNOWN) return LZMA_PROG_ERROR; + size_t filter_count = 0; do { // There can be a maximum of four filters. if (filter_count == LZMA_FILTERS_MAX) diff --git a/Utilities/cmliblzma/liblzma/common/block_util.c b/Utilities/cmliblzma/liblzma/common/block_util.c index 4cd34d1..00c7fe8 100644 --- a/Utilities/cmliblzma/liblzma/common/block_util.c +++ b/Utilities/cmliblzma/liblzma/common/block_util.c @@ -17,14 +17,11 @@ extern LZMA_API(lzma_ret) lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) { - uint32_t container_size; - lzma_vli compressed_size; - // Validate everything but Uncompressed Size and filters. if (lzma_block_unpadded_size(block) == 0) return LZMA_PROG_ERROR; - container_size = block->header_size + const uint32_t container_size = block->header_size + lzma_check_size(block->check); // Validate that Compressed Size will be greater than zero. @@ -34,7 +31,7 @@ lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) // Calculate what Compressed Size is supposed to be. // If Compressed Size was present in Block Header, // compare that the new value matches it. - compressed_size = unpadded_size - container_size; + const lzma_vli compressed_size = unpadded_size - container_size; if (block->compressed_size != LZMA_VLI_UNKNOWN && block->compressed_size != compressed_size) return LZMA_DATA_ERROR; @@ -48,15 +45,13 @@ lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) extern LZMA_API(lzma_vli) lzma_block_unpadded_size(const lzma_block *block) { - lzma_vli unpadded_size; - // Validate the values that we are interested in i.e. all but // Uncompressed Size and the filters. // // NOTE: This function is used for validation too, so it is // essential that these checks are always done even if // Compressed Size is unknown. - if (block == NULL || block->version != 0 + if (block == NULL || block->version > 1 || block->header_size < LZMA_BLOCK_HEADER_SIZE_MIN || block->header_size > LZMA_BLOCK_HEADER_SIZE_MAX || (block->header_size & 3) @@ -71,7 +66,7 @@ lzma_block_unpadded_size(const lzma_block *block) return LZMA_VLI_UNKNOWN; // Calculate Unpadded Size and validate it. - unpadded_size = block->compressed_size + const lzma_vli unpadded_size = block->compressed_size + block->header_size + lzma_check_size(block->check); diff --git a/Utilities/cmliblzma/liblzma/common/common.c b/Utilities/cmliblzma/liblzma/common/common.c index 2e723c8..57e3f8e 100644 --- a/Utilities/cmliblzma/liblzma/common/common.c +++ b/Utilities/cmliblzma/liblzma/common/common.c @@ -36,14 +36,14 @@ lzma_version_string(void) /////////////////////// extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) -lzma_alloc(size_t size, lzma_allocator *allocator) +lzma_alloc(size_t size, const lzma_allocator *allocator) { - void *ptr; - // Some malloc() variants return NULL if called with size == 0. if (size == 0) size = 1; + void *ptr; + if (allocator != NULL && allocator->alloc != NULL) ptr = allocator->alloc(allocator->opaque, 1, size); else @@ -53,8 +53,29 @@ lzma_alloc(size_t size, lzma_allocator *allocator) } +extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) +lzma_alloc_zero(size_t size, const lzma_allocator *allocator) +{ + // Some calloc() variants return NULL if called with size == 0. + if (size == 0) + size = 1; + + void *ptr; + + if (allocator != NULL && allocator->alloc != NULL) { + ptr = allocator->alloc(allocator->opaque, 1, size); + if (ptr != NULL) + memzero(ptr, size); + } else { + ptr = calloc(1, size); + } + + return ptr; +} + + extern void -lzma_free(void *ptr, lzma_allocator *allocator) +lzma_free(void *ptr, const lzma_allocator *allocator) { if (allocator != NULL && allocator->free != NULL) allocator->free(allocator->opaque, ptr); @@ -70,9 +91,9 @@ lzma_free(void *ptr, lzma_allocator *allocator) ////////// extern size_t -lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size) +lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) { const size_t in_avail = in_size - *in_pos; const size_t out_avail = out_size - *out_pos; @@ -88,7 +109,7 @@ lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, extern lzma_ret -lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_next_filter_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { lzma_next_coder_init(filters[0].init, next, allocator); @@ -99,7 +120,7 @@ lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_next_filter_update(lzma_next_coder *next, lzma_allocator *allocator, +lzma_next_filter_update(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *reversed_filters) { // Check that the application isn't trying to change the Filter ID. @@ -117,7 +138,7 @@ lzma_next_filter_update(lzma_next_coder *next, lzma_allocator *allocator, extern void -lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator) +lzma_next_end(lzma_next_coder *next, const lzma_allocator *allocator) { if (next->init != (uintptr_t)(NULL)) { // To avoid tiny end functions that simply call @@ -156,10 +177,8 @@ lzma_strm_init(lzma_stream *strm) strm->internal->next = LZMA_NEXT_CODER_INIT; } - strm->internal->supported_actions[LZMA_RUN] = false; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = false; - strm->internal->supported_actions[LZMA_FULL_FLUSH] = false; - strm->internal->supported_actions[LZMA_FINISH] = false; + memzero(strm->internal->supported_actions, + sizeof(strm->internal->supported_actions)); strm->internal->sequence = ISEQ_RUN; strm->internal->allow_buf_error = false; @@ -173,16 +192,12 @@ lzma_strm_init(lzma_stream *strm) extern LZMA_API(lzma_ret) lzma_code(lzma_stream *strm, lzma_action action) { - size_t in_pos = 0; - size_t out_pos = 0; - lzma_ret ret; - // Sanity checks if ((strm->next_in == NULL && strm->avail_in != 0) || (strm->next_out == NULL && strm->avail_out != 0) || strm->internal == NULL || strm->internal->next.code == NULL - || (unsigned int)(action) > LZMA_FINISH + || (unsigned int)(action) > LZMA_ACTION_MAX || !strm->internal->supported_actions[action]) return LZMA_PROG_ERROR; @@ -217,6 +232,10 @@ lzma_code(lzma_stream *strm, lzma_action action) case LZMA_FINISH: strm->internal->sequence = ISEQ_FINISH; break; + + case LZMA_FULL_BARRIER: + strm->internal->sequence = ISEQ_FULL_BARRIER; + break; } break; @@ -244,6 +263,13 @@ lzma_code(lzma_stream *strm, lzma_action action) break; + case ISEQ_FULL_BARRIER: + if (action != LZMA_FULL_BARRIER + || strm->internal->avail_in != strm->avail_in) + return LZMA_PROG_ERROR; + + break; + case ISEQ_END: return LZMA_STREAM_END; @@ -252,7 +278,9 @@ lzma_code(lzma_stream *strm, lzma_action action) return LZMA_PROG_ERROR; } - ret = strm->internal->next.code( + size_t in_pos = 0; + size_t out_pos = 0; + lzma_ret ret = strm->internal->next.code( strm->internal->next.coder, strm->allocator, strm->next_in, &in_pos, strm->avail_in, strm->next_out, &out_pos, strm->avail_out, action); @@ -267,7 +295,9 @@ lzma_code(lzma_stream *strm, lzma_action action) strm->internal->avail_in = strm->avail_in; - switch (ret) { + // Cast is needed to silence a warning about LZMA_TIMED_OUT, which + // isn't part of lzma_ret enumeration. + switch ((unsigned int)(ret)) { case LZMA_OK: // Don't return LZMA_BUF_ERROR when it happens the first time. // This is to avoid returning LZMA_BUF_ERROR when avail_out @@ -283,9 +313,16 @@ lzma_code(lzma_stream *strm, lzma_action action) } break; + case LZMA_TIMED_OUT: + strm->internal->allow_buf_error = false; + ret = LZMA_OK; + break; + case LZMA_STREAM_END: if (strm->internal->sequence == ISEQ_SYNC_FLUSH - || strm->internal->sequence == ISEQ_FULL_FLUSH) + || strm->internal->sequence == ISEQ_FULL_FLUSH + || strm->internal->sequence + == ISEQ_FULL_BARRIER) strm->internal->sequence = ISEQ_RUN; else strm->internal->sequence = ISEQ_END; @@ -325,6 +362,22 @@ lzma_end(lzma_stream *strm) } +extern LZMA_API(void) +lzma_get_progress(lzma_stream *strm, + uint64_t *progress_in, uint64_t *progress_out) +{ + if (strm->internal->next.get_progress != NULL) { + strm->internal->next.get_progress(strm->internal->next.coder, + progress_in, progress_out); + } else { + *progress_in = strm->total_in; + *progress_out = strm->total_out; + } + + return; +} + + extern LZMA_API(lzma_check) lzma_get_check(const lzma_stream *strm) { @@ -382,8 +435,10 @@ lzma_memlimit_set(lzma_stream *strm, uint64_t new_memlimit) || strm->internal->next.memconfig == NULL) return LZMA_PROG_ERROR; - if (new_memlimit != 0 && new_memlimit < LZMA_MEMUSAGE_BASE) - return LZMA_MEMLIMIT_ERROR; + // Zero is a special value that cannot be used as an actual limit. + // If 0 was specified, use 1 instead. + if (new_memlimit == 0) + new_memlimit = 1; return strm->internal->next.memconfig(strm->internal->next.coder, &memusage, &old_memlimit, new_memlimit); diff --git a/Utilities/cmliblzma/liblzma/common/common.h b/Utilities/cmliblzma/liblzma/common/common.h index a6a2818..dde3ae0 100644 --- a/Utilities/cmliblzma/liblzma/common/common.h +++ b/Utilities/cmliblzma/liblzma/common/common.h @@ -48,6 +48,13 @@ #define LZMA_BUFFER_SIZE 4096 +/// Maximum number of worker threads within one multithreaded component. +/// The limit exists solely to make it simpler to prevent integer overflows +/// when allocating structures etc. This should be big enough for now... +/// the code won't scale anywhere close to this number anyway. +#define LZMA_THREADS_MAX 16384 + + /// Starting value for memory usage estimates. Instead of calculating size /// of _every_ structure and taking into account malloc() overhead etc., we /// add a base size to all memory usage estimates. It's not very accurate @@ -65,12 +72,20 @@ ( LZMA_TELL_NO_CHECK \ | LZMA_TELL_UNSUPPORTED_CHECK \ | LZMA_TELL_ANY_CHECK \ + | LZMA_IGNORE_CHECK \ | LZMA_CONCATENATED ) -/// Type of encoder/decoder specific data; the actual structure is defined -/// differently in different coders. -typedef struct lzma_coder_s lzma_coder; +/// Largest valid lzma_action value as unsigned integer. +#define LZMA_ACTION_MAX ((unsigned int)(LZMA_FULL_BARRIER)) + + +/// Special return value (lzma_ret) to indicate that a timeout was reached +/// and lzma_code() must not return LZMA_BUF_ERROR. This is converted to +/// LZMA_OK in lzma_code(). This is not in the lzma_ret enumeration because +/// there's no need to have it in the public API. +#define LZMA_TIMED_OUT 32 + typedef struct lzma_next_coder_s lzma_next_coder; @@ -79,7 +94,7 @@ typedef struct lzma_filter_info_s lzma_filter_info; /// Type of a function used to initialize a filter encoder or decoder typedef lzma_ret (*lzma_init_function)( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); /// Type of a function to do some kind of coding work (filters, Stream, @@ -87,15 +102,15 @@ typedef lzma_ret (*lzma_init_function)( /// input and output buffers, but for simplicity they still use this same /// function prototype. typedef lzma_ret (*lzma_code_function)( - lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, + void *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action); /// Type of a function to free the memory allocated for the coder typedef void (*lzma_end_function)( - lzma_coder *coder, lzma_allocator *allocator); + void *coder, const lzma_allocator *allocator); /// Raw coder validates and converts an array of lzma_filter structures to @@ -118,7 +133,7 @@ struct lzma_filter_info_s { /// Hold data and function pointers of the next filter in the chain. struct lzma_next_coder_s { /// Pointer to coder-specific data - lzma_coder *coder; + void *coder; /// Filter ID. This is LZMA_VLI_UNKNOWN when this structure doesn't /// point to a filter coder. @@ -138,35 +153,41 @@ struct lzma_next_coder_s { /// lzma_next_coder.coder. lzma_end_function end; + /// Pointer to a function to get progress information. If this is NULL, + /// lzma_stream.total_in and .total_out are used instead. + void (*get_progress)(void *coder, + uint64_t *progress_in, uint64_t *progress_out); + /// Pointer to function to return the type of the integrity check. /// Most coders won't support this. - lzma_check (*get_check)(const lzma_coder *coder); + lzma_check (*get_check)(const void *coder); /// Pointer to function to get and/or change the memory usage limit. /// If new_memlimit == 0, the limit is not changed. - lzma_ret (*memconfig)(lzma_coder *coder, uint64_t *memusage, + lzma_ret (*memconfig)(void *coder, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit); /// Update the filter-specific options or the whole filter chain /// in the encoder. - lzma_ret (*update)(lzma_coder *coder, lzma_allocator *allocator, + lzma_ret (*update)(void *coder, const lzma_allocator *allocator, const lzma_filter *filters, const lzma_filter *reversed_filters); }; -/// Constant to initialize lzma_next_coder structure -static const lzma_next_coder LZMA_NEXT_CODER_INIT = - { - NULL, - LZMA_VLI_UNKNOWN, - (uintptr_t)(NULL), - NULL, - NULL, - NULL, - NULL, - NULL, - }; +/// Macro to initialize lzma_next_coder structure +#define LZMA_NEXT_CODER_INIT \ + (lzma_next_coder){ \ + .coder = NULL, \ + .init = (uintptr_t)(NULL), \ + .id = LZMA_VLI_UNKNOWN, \ + .code = NULL, \ + .end = NULL, \ + .get_progress = NULL, \ + .get_check = NULL, \ + .memconfig = NULL, \ + .update = NULL, \ + } /// Internal data for lzma_strm_init, lzma_code, and lzma_end. A pointer to @@ -184,6 +205,7 @@ struct lzma_internal_s { ISEQ_SYNC_FLUSH, ISEQ_FULL_FLUSH, ISEQ_FINISH, + ISEQ_FULL_BARRIER, ISEQ_END, ISEQ_ERROR, } sequence; @@ -194,7 +216,7 @@ struct lzma_internal_s { size_t avail_in; /// Indicates which lzma_action values are allowed by next.code. - bool supported_actions[4]; + bool supported_actions[LZMA_ACTION_MAX + 1]; /// If true, lzma_code will return LZMA_BUF_ERROR if no progress was /// made (no input consumed and no output produced by next.code). @@ -203,15 +225,21 @@ struct lzma_internal_s { /// Allocates memory -extern void *lzma_alloc(size_t size, lzma_allocator *allocator) +extern void *lzma_alloc(size_t size, const lzma_allocator *allocator) lzma_attribute((__malloc__)) lzma_attr_alloc_size(1); +/// Allocates memory and zeroes it (like calloc()). This can be faster +/// than lzma_alloc() + memzero() while being backward compatible with +/// custom allocators. +extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) + lzma_alloc_zero(size_t size, const lzma_allocator *allocator); + /// Frees memory -extern void lzma_free(void *ptr, lzma_allocator *allocator); +extern void lzma_free(void *ptr, const lzma_allocator *allocator); /// Allocates strm->internal if it is NULL, and initializes *strm and -/// strm->internal. This function is only called via lzma_next_strm_init2 macro. +/// strm->internal. This function is only called via lzma_next_strm_init macro. extern lzma_ret lzma_strm_init(lzma_stream *strm); /// Initializes the next filter in the chain, if any. This takes care of @@ -219,24 +247,26 @@ extern lzma_ret lzma_strm_init(lzma_stream *strm); /// than the filter being initialized now. This way the actual filter /// initialization functions don't need to use lzma_next_coder_init macro. extern lzma_ret lzma_next_filter_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); /// Update the next filter in the chain, if any. This checks that /// the application is not trying to change the Filter IDs. extern lzma_ret lzma_next_filter_update( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *reversed_filters); /// Frees the memory allocated for next->coder either using next->end or, /// if next->end is NULL, using lzma_free. -extern void lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator); +extern void lzma_next_end(lzma_next_coder *next, + const lzma_allocator *allocator); /// Copy as much data as possible from in[] to out[] and update *in_pos /// and *out_pos accordingly. Returns the number of bytes copied. -extern size_t lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size); +extern size_t lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); /// \brief Return if expression doesn't evaluate to LZMA_OK @@ -269,37 +299,15 @@ do { \ /// (The function being called will use lzma_next_coder_init()). If /// initialization fails, memory that wasn't freed by func() is freed /// along strm->internal. -#define lzma_next_strm_init1(func, strm, arg1) \ +#define lzma_next_strm_init(func, strm, ...) \ do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ -} while (0) - -#define lzma_next_strm_init2(func, strm, arg1, arg2) \ -do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1, arg2); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ -} while (0) - -#define lzma_next_strm_init3(func, strm, arg1, arg2, arg3) \ -do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1, arg2, arg3); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ + return_if_error(lzma_strm_init(strm)); \ + const lzma_ret ret_ = func(&(strm)->internal->next, \ + (strm)->allocator, __VA_ARGS__); \ + if (ret_ != LZMA_OK) { \ + lzma_end(strm); \ + return ret_; \ + } \ } while (0) #endif diff --git a/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c index c4be34c..48eb56f 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c @@ -15,8 +15,8 @@ extern LZMA_API(lzma_ret) lzma_easy_buffer_encode(uint32_t preset, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, - uint8_t *out, size_t *out_pos, size_t out_size) + const lzma_allocator *allocator, const uint8_t *in, + size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { lzma_options_easy opt_easy; if (lzma_easy_preset(&opt_easy, preset)) diff --git a/Utilities/cmliblzma/liblzma/common/easy_encoder.c b/Utilities/cmliblzma/liblzma/common/easy_encoder.c index d13ccd7..5cb492d 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/easy_encoder.c @@ -11,7 +11,6 @@ /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" -#include "stream_encoder.h" extern LZMA_API(lzma_ret) diff --git a/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c index 65665c1..6620986 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c @@ -14,30 +14,27 @@ extern LZMA_API(lzma_ret) -lzma_raw_buffer_decode(const lzma_filter *filters, lzma_allocator *allocator, +lzma_raw_buffer_decode( + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder next = LZMA_NEXT_CODER_INIT; - size_t in_start; - size_t out_start; - lzma_ret ret; - // Validate what isn't validated later in filter_common.c. if (in == NULL || in_pos == NULL || *in_pos > in_size || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; // Initialize the decoer. + lzma_next_coder next = LZMA_NEXT_CODER_INIT; return_if_error(lzma_raw_decoder_init(&next, allocator, filters)); // Store the positions so that we can restore them if something // goes wrong. - in_start = *in_pos; - out_start = *out_pos; + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; // Do the actual decoding and free decoder's memory. - ret = next.code(next.coder, allocator, in, in_pos, in_size, + lzma_ret ret = next.code(next.coder, allocator, in, in_pos, in_size, out, out_pos, out_size, LZMA_FINISH); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c index b23329f..dda18e3 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c @@ -14,29 +14,27 @@ extern LZMA_API(lzma_ret) -lzma_raw_buffer_encode(const lzma_filter *filters, lzma_allocator *allocator, - const uint8_t *in, size_t in_size, uint8_t *out, - size_t *out_pos, size_t out_size) +lzma_raw_buffer_encode( + const lzma_filter *filters, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder next = LZMA_NEXT_CODER_INIT; - size_t out_start; - size_t in_pos = 0; - lzma_ret ret; - // Validate what isn't validated later in filter_common.c. if ((in == NULL && in_size != 0) || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; // Initialize the encoder + lzma_next_coder next = LZMA_NEXT_CODER_INIT; return_if_error(lzma_raw_encoder_init(&next, allocator, filters)); // Store the output position so that we can restore it if // something goes wrong. - out_start = *out_pos; + const size_t out_start = *out_pos; // Do the actual encoding and free coder's memory. - ret = next.code(next.coder, allocator, in, &in_pos, in_size, + size_t in_pos = 0; + lzma_ret ret = next.code(next.coder, allocator, in, &in_pos, in_size, out, out_pos, out_size, LZMA_FINISH); lzma_next_end(&next, allocator); diff --git a/Utilities/cmliblzma/liblzma/common/filter_common.c b/Utilities/cmliblzma/liblzma/common/filter_common.c index d2b9e08..9ad5d5d 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_common.c +++ b/Utilities/cmliblzma/liblzma/common/filter_common.c @@ -36,101 +36,100 @@ static const struct { } features[] = { #if defined (HAVE_ENCODER_LZMA1) || defined(HAVE_DECODER_LZMA1) { - LZMA_FILTER_LZMA1, - sizeof(lzma_options_lzma), - false, - true, - true, + .id = LZMA_FILTER_LZMA1, + .options_size = sizeof(lzma_options_lzma), + .non_last_ok = false, + .last_ok = true, + .changes_size = true, }, #endif #if defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2) { - LZMA_FILTER_LZMA2, - sizeof(lzma_options_lzma), - false, - true, - true, + .id = LZMA_FILTER_LZMA2, + .options_size = sizeof(lzma_options_lzma), + .non_last_ok = false, + .last_ok = true, + .changes_size = true, }, #endif #if defined(HAVE_ENCODER_X86) || defined(HAVE_DECODER_X86) { - LZMA_FILTER_X86, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_X86, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_POWERPC) || defined(HAVE_DECODER_POWERPC) { - LZMA_FILTER_POWERPC, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_POWERPC, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_IA64) || defined(HAVE_DECODER_IA64) { - LZMA_FILTER_IA64, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_IA64, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_ARM) || defined(HAVE_DECODER_ARM) { - LZMA_FILTER_ARM, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_ARM, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_ARMTHUMB) || defined(HAVE_DECODER_ARMTHUMB) { - LZMA_FILTER_ARMTHUMB, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_ARMTHUMB, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_SPARC) || defined(HAVE_DECODER_SPARC) { - LZMA_FILTER_SPARC, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_SPARC, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) { - LZMA_FILTER_DELTA, - sizeof(lzma_options_delta), - true, - false, - false, + .id = LZMA_FILTER_DELTA, + .options_size = sizeof(lzma_options_delta), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif { - LZMA_VLI_UNKNOWN + .id = LZMA_VLI_UNKNOWN } }; extern LZMA_API(lzma_ret) lzma_filters_copy(const lzma_filter *src, lzma_filter *dest, - lzma_allocator *allocator) + const lzma_allocator *allocator) { - size_t i; - lzma_ret ret; - if (src == NULL || dest == NULL) return LZMA_PROG_ERROR; + lzma_ret ret; + size_t i; for (i = 0; src[i].id != LZMA_VLI_UNKNOWN; ++i) { // There must be a maximum of four filters plus // the array terminator. @@ -194,6 +193,10 @@ error: static lzma_ret validate_chain(const lzma_filter *filters, size_t *count) { + // There must be at least one filter. + if (filters == NULL || filters[0].id == LZMA_VLI_UNKNOWN) + return LZMA_PROG_ERROR; + // Number of non-last filters that may change the size of the data // significantly (that is, more than 1-2 % or so). size_t changes_size_count = 0; @@ -207,11 +210,6 @@ validate_chain(const lzma_filter *filters, size_t *count) bool last_ok = false; size_t i = 0; - - // There must be at least one filter. - if (filters == NULL || filters[0].id == LZMA_VLI_UNKNOWN) - return LZMA_PROG_ERROR; - do { size_t j; for (j = 0; filters[i].id != features[j].id; ++j) @@ -241,21 +239,18 @@ validate_chain(const lzma_filter *filters, size_t *count) extern lzma_ret -lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options, lzma_filter_find coder_find, bool is_encoder) { - lzma_filter_info filters[LZMA_FILTERS_MAX + 1]; - size_t count; - size_t i; - lzma_ret ret; - // Do some basic validation and get the number of filters. + size_t count; return_if_error(validate_chain(options, &count)); // Set the filter functions and copy the options pointer. + lzma_filter_info filters[LZMA_FILTERS_MAX + 1]; if (is_encoder) { - for (i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { // The order of the filters is reversed in the // encoder. It allows more efficient handling // of the uncompressed data. @@ -271,7 +266,7 @@ lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, filters[j].options = options[i].options; } } else { - for (i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { const lzma_filter_coder *const fc = coder_find(options[i].id); if (fc == NULL || fc->init == NULL) @@ -288,7 +283,7 @@ lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, filters[count].init = NULL; // Initialize the filters. - ret = lzma_next_filter_init(next, allocator, filters); + const lzma_ret ret = lzma_next_filter_init(next, allocator, filters); if (ret != LZMA_OK) lzma_next_end(next, allocator); @@ -300,9 +295,6 @@ extern uint64_t lzma_raw_coder_memusage(lzma_filter_find coder_find, const lzma_filter *filters) { - uint64_t total = 0; - size_t i = 0; - // The chain has to have at least one filter. { size_t tmp; @@ -310,6 +302,9 @@ lzma_raw_coder_memusage(lzma_filter_find coder_find, return UINT64_MAX; } + uint64_t total = 0; + size_t i = 0; + do { const lzma_filter_coder *const fc = coder_find(filters[i].id); diff --git a/Utilities/cmliblzma/liblzma/common/filter_common.h b/Utilities/cmliblzma/liblzma/common/filter_common.h index cd61fc0..42a26a2 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_common.h +++ b/Utilities/cmliblzma/liblzma/common/filter_common.h @@ -36,7 +36,7 @@ typedef const lzma_filter_coder *(*lzma_filter_find)(lzma_vli id); extern lzma_ret lzma_raw_coder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters, lzma_filter_find coder_find, bool is_encoder); diff --git a/Utilities/cmliblzma/liblzma/common/filter_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_decoder.c index cce2b30..c75b0a8 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_decoder.c @@ -35,7 +35,8 @@ typedef struct { /// \return - LZMA_OK: Properties decoded successfully. /// - LZMA_OPTIONS_ERROR: Unsupported properties /// - LZMA_MEM_ERROR: Memory allocation failed. - lzma_ret (*props_decode)(void **options, lzma_allocator *allocator, + lzma_ret (*props_decode)( + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); } lzma_filter_decoder; @@ -44,74 +45,74 @@ typedef struct { static const lzma_filter_decoder decoders[] = { #ifdef HAVE_DECODER_LZMA1 { - LZMA_FILTER_LZMA1, - &lzma_lzma_decoder_init, - &lzma_lzma_decoder_memusage, - &lzma_lzma_props_decode, + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_decoder_init, + .memusage = &lzma_lzma_decoder_memusage, + .props_decode = &lzma_lzma_props_decode, }, #endif #ifdef HAVE_DECODER_LZMA2 { - LZMA_FILTER_LZMA2, - &lzma_lzma2_decoder_init, - &lzma_lzma2_decoder_memusage, - &lzma_lzma2_props_decode, + .id = LZMA_FILTER_LZMA2, + .init = &lzma_lzma2_decoder_init, + .memusage = &lzma_lzma2_decoder_memusage, + .props_decode = &lzma_lzma2_props_decode, }, #endif #ifdef HAVE_DECODER_X86 { - LZMA_FILTER_X86, - &lzma_simple_x86_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_X86, + .init = &lzma_simple_x86_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_POWERPC { - LZMA_FILTER_POWERPC, - &lzma_simple_powerpc_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_POWERPC, + .init = &lzma_simple_powerpc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_IA64 { - LZMA_FILTER_IA64, - &lzma_simple_ia64_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_IA64, + .init = &lzma_simple_ia64_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_ARM { - LZMA_FILTER_ARM, - &lzma_simple_arm_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_ARM, + .init = &lzma_simple_arm_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_ARMTHUMB { - LZMA_FILTER_ARMTHUMB, - &lzma_simple_armthumb_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_ARMTHUMB, + .init = &lzma_simple_armthumb_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_SPARC { - LZMA_FILTER_SPARC, - &lzma_simple_sparc_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_SPARC, + .init = &lzma_simple_sparc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_DELTA { - LZMA_FILTER_DELTA, - &lzma_delta_decoder_init, - &lzma_delta_coder_memusage, - &lzma_delta_props_decode, + .id = LZMA_FILTER_DELTA, + .init = &lzma_delta_decoder_init, + .memusage = &lzma_delta_coder_memusage, + .props_decode = &lzma_delta_props_decode, }, #endif }; @@ -120,8 +121,7 @@ static const lzma_filter_decoder decoders[] = { static const lzma_filter_decoder * decoder_find(lzma_vli id) { - size_t i; - for (i = 0; i < ARRAY_SIZE(decoders); ++i) + for (size_t i = 0; i < ARRAY_SIZE(decoders); ++i) if (decoders[i].id == id) return decoders + i; @@ -137,7 +137,7 @@ lzma_filter_decoder_is_supported(lzma_vli id) extern lzma_ret -lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options) { return lzma_raw_coder_init(next, allocator, @@ -148,7 +148,7 @@ lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_raw_decoder(lzma_stream *strm, const lzma_filter *options) { - lzma_next_strm_init1(lzma_raw_decoder_init, strm, options); + lzma_next_strm_init(lzma_raw_decoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -166,14 +166,13 @@ lzma_raw_decoder_memusage(const lzma_filter *filters) extern LZMA_API(lzma_ret) -lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator, +lzma_properties_decode(lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - const lzma_filter_decoder *const fd = decoder_find(filter->id); - // Make it always NULL so that the caller can always safely free() it. filter->options = NULL; + const lzma_filter_decoder *const fd = decoder_find(filter->id); if (fd == NULL) return LZMA_OPTIONS_ERROR; diff --git a/Utilities/cmliblzma/liblzma/common/filter_decoder.h b/Utilities/cmliblzma/liblzma/common/filter_decoder.h index d5c68bd..a2e255f 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/filter_decoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_raw_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options); #endif diff --git a/Utilities/cmliblzma/liblzma/common/filter_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_encoder.c index 9fdb100..c5d8f39 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_encoder.c @@ -30,11 +30,11 @@ typedef struct { /// invalid, UINT64_MAX is returned. uint64_t (*memusage)(const void *options); - /// Calculates the minimum sane size for Blocks (or other types of - /// chunks) to which the input data can be split to make - /// multithreaded encoding possible. If this is NULL, it is assumed - /// that the encoder is fast enough with single thread. - lzma_vli (*chunk_size)(const void *options); + /// Calculates the recommended Uncompressed Size for .xz Blocks to + /// which the input data can be split to make multithreaded + /// encoding possible. If this is NULL, it is assumed that + /// the encoder is fast enough with single thread. + uint64_t (*block_size)(const void *options); /// Tells the size of the Filter Properties field. If options are /// invalid, UINT32_MAX is returned. If this is NULL, props_size_fixed @@ -56,101 +56,95 @@ typedef struct { static const lzma_filter_encoder encoders[] = { #ifdef HAVE_ENCODER_LZMA1 { - LZMA_FILTER_LZMA1, - &lzma_lzma_encoder_init, - &lzma_lzma_encoder_memusage, - NULL, // FIXME - NULL, - 5, - &lzma_lzma_props_encode, + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_encoder_init, + .memusage = &lzma_lzma_encoder_memusage, + .block_size = NULL, // FIXME + .props_size_get = NULL, + .props_size_fixed = 5, + .props_encode = &lzma_lzma_props_encode, }, #endif #ifdef HAVE_ENCODER_LZMA2 { - LZMA_FILTER_LZMA2, - &lzma_lzma2_encoder_init, - &lzma_lzma2_encoder_memusage, - NULL, // FIXME - NULL, - 1, - &lzma_lzma2_props_encode, + .id = LZMA_FILTER_LZMA2, + .init = &lzma_lzma2_encoder_init, + .memusage = &lzma_lzma2_encoder_memusage, + .block_size = &lzma_lzma2_block_size, // FIXME + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_lzma2_props_encode, }, #endif #ifdef HAVE_ENCODER_X86 { - LZMA_FILTER_X86, - &lzma_simple_x86_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_X86, + .init = &lzma_simple_x86_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_POWERPC { - LZMA_FILTER_POWERPC, - &lzma_simple_powerpc_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_POWERPC, + .init = &lzma_simple_powerpc_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_IA64 { - LZMA_FILTER_IA64, - &lzma_simple_ia64_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_IA64, + .init = &lzma_simple_ia64_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_ARM { - LZMA_FILTER_ARM, - &lzma_simple_arm_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_ARM, + .init = &lzma_simple_arm_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_ARMTHUMB { - LZMA_FILTER_ARMTHUMB, - &lzma_simple_armthumb_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_ARMTHUMB, + .init = &lzma_simple_armthumb_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_SPARC { - LZMA_FILTER_SPARC, - &lzma_simple_sparc_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_SPARC, + .init = &lzma_simple_sparc_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_DELTA { - LZMA_FILTER_DELTA, - &lzma_delta_encoder_init, - &lzma_delta_coder_memusage, - NULL, - NULL, - 1, - &lzma_delta_props_encode, + .id = LZMA_FILTER_DELTA, + .init = &lzma_delta_encoder_init, + .memusage = &lzma_delta_coder_memusage, + .block_size = NULL, + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_delta_props_encode, }, #endif }; @@ -159,8 +153,7 @@ static const lzma_filter_encoder encoders[] = { static const lzma_filter_encoder * encoder_find(lzma_vli id) { - size_t i; - for (i = 0; i < ARRAY_SIZE(encoders); ++i) + for (size_t i = 0; i < ARRAY_SIZE(encoders); ++i) if (encoders[i].id == id) return encoders + i; @@ -178,10 +171,6 @@ lzma_filter_encoder_is_supported(lzma_vli id) extern LZMA_API(lzma_ret) lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) { - size_t i; - size_t count = 1; - lzma_filter reversed_filters[LZMA_FILTERS_MAX + 1]; - if (strm->internal->next.update == NULL) return LZMA_PROG_ERROR; @@ -191,10 +180,12 @@ lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) // The actual filter chain in the encoder is reversed. Some things // still want the normal order chain, so we provide both. + size_t count = 1; while (filters[count].id != LZMA_VLI_UNKNOWN) ++count; - for (i = 0; i < count; ++i) + lzma_filter reversed_filters[LZMA_FILTERS_MAX + 1]; + for (size_t i = 0; i < count; ++i) reversed_filters[count - i - 1] = filters[i]; reversed_filters[count].id = LZMA_VLI_UNKNOWN; @@ -205,7 +196,7 @@ lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) extern lzma_ret -lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options) { return lzma_raw_coder_init(next, allocator, @@ -216,7 +207,7 @@ lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options) { - lzma_next_strm_init3(lzma_raw_coder_init, strm, options, + lzma_next_strm_init(lzma_raw_coder_init, strm, options, (lzma_filter_find)(&encoder_find), true); strm->internal->supported_actions[LZMA_RUN] = true; @@ -235,20 +226,19 @@ lzma_raw_encoder_memusage(const lzma_filter *filters) } -/* -extern LZMA_API(lzma_vli) -lzma_chunk_size(const lzma_filter *filters) +extern uint64_t +lzma_mt_block_size(const lzma_filter *filters) { - lzma_vli max = 0; + uint64_t max = 0; for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { const lzma_filter_encoder *const fe = encoder_find(filters[i].id); - if (fe->chunk_size != NULL) { - const lzma_vli size - = fe->chunk_size(filters[i].options); - if (size == LZMA_VLI_UNKNOWN) - return LZMA_VLI_UNKNOWN; + if (fe->block_size != NULL) { + const uint64_t size + = fe->block_size(filters[i].options); + if (size == 0) + return 0; if (size > max) max = size; @@ -257,7 +247,6 @@ lzma_chunk_size(const lzma_filter *filters) return max; } -*/ extern LZMA_API(lzma_ret) diff --git a/Utilities/cmliblzma/liblzma/common/filter_encoder.h b/Utilities/cmliblzma/liblzma/common/filter_encoder.h index 5bc137f..f1d5683 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/filter_encoder.h @@ -16,12 +16,12 @@ #include "common.h" -// FIXME: Might become a part of the public API once finished. -// extern lzma_vli lzma_chunk_size(const lzma_filter *filters); +// FIXME: Might become a part of the public API. +extern uint64_t lzma_mt_block_size(const lzma_filter *filters); extern lzma_ret lzma_raw_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c index aa2dbd5..ddfb085 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c @@ -15,12 +15,9 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { - lzma_vli props_size; - lzma_ret ret; - // Set the pointer to NULL so the caller can always safely free it. filter->options = NULL; @@ -32,6 +29,7 @@ lzma_filter_flags_decode( return LZMA_DATA_ERROR; // Size of Properties + lzma_vli props_size; return_if_error(lzma_vli_decode(&props_size, NULL, in, in_pos, in_size)); @@ -39,7 +37,7 @@ lzma_filter_flags_decode( if (in_size - *in_pos < props_size) return LZMA_DATA_ERROR; - ret = lzma_properties_decode( + const lzma_ret ret = lzma_properties_decode( filter, allocator, in + *in_pos, props_size); *in_pos += props_size; diff --git a/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c index 755c407..d110566 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c @@ -31,8 +31,6 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filter, uint8_t *out, size_t *out_pos, size_t out_size) { - uint32_t props_size; - // Filter ID if (filter->id >= LZMA_FILTER_RESERVED_START) return LZMA_PROG_ERROR; @@ -41,6 +39,7 @@ lzma_filter_flags_encode(const lzma_filter *filter, out, out_pos, out_size)); // Size of Properties + uint32_t props_size; return_if_error(lzma_properties_size(&props_size, filter)); return_if_error(lzma_vli_encode(props_size, NULL, out, out_pos, out_size)); diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder.h b/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c index 46a7aed..f468366 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file stream_encoder.h -/// \brief Encodes .xz Streams +/// \file hardware_cputhreads.c +/// \brief Get the number of CPU threads or cores // // Author: Lasse Collin // @@ -10,14 +10,13 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_STREAM_ENCODER_H -#define LZMA_STREAM_ENCODER_H - #include "common.h" +#include "tuklib_cpucores.h" -extern lzma_ret lzma_stream_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, - const lzma_filter *filters, lzma_check check); -#endif +extern LZMA_API(uint32_t) +lzma_cputhreads(void) +{ + return tuklib_cpucores(); +} diff --git a/Utilities/cmliblzma/liblzma/common/index.c b/Utilities/cmliblzma/liblzma/common/index.c index 26135d2..26e4e51 100644 --- a/Utilities/cmliblzma/liblzma/common/index.c +++ b/Utilities/cmliblzma/liblzma/common/index.c @@ -191,8 +191,8 @@ index_tree_init(index_tree *tree) /// Helper for index_tree_end() static void -index_tree_node_end(index_tree_node *node, lzma_allocator *allocator, - void (*free_func)(void *node, lzma_allocator *allocator)) +index_tree_node_end(index_tree_node *node, const lzma_allocator *allocator, + void (*free_func)(void *node, const lzma_allocator *allocator)) { // The tree won't ever be very huge, so recursion should be fine. // 20 levels in the tree is likely quite a lot already in practice. @@ -202,22 +202,21 @@ index_tree_node_end(index_tree_node *node, lzma_allocator *allocator, if (node->right != NULL) index_tree_node_end(node->right, allocator, free_func); - if (free_func != NULL) - free_func(node, allocator); - - lzma_free(node, allocator); + free_func(node, allocator); return; } -/// Free the meory allocated for a tree. If free_func is not NULL, -/// it is called on each node before freeing the node. This is used -/// to free the Record groups from each index_stream before freeing -/// the index_stream itself. +/// Free the memory allocated for a tree. Each node is freed using the +/// given free_func which is either &lzma_free or &index_stream_end. +/// The latter is used to free the Record groups from each index_stream +/// before freeing the index_stream itself. static void -index_tree_end(index_tree *tree, lzma_allocator *allocator, - void (*free_func)(void *node, lzma_allocator *allocator)) +index_tree_end(index_tree *tree, const lzma_allocator *allocator, + void (*free_func)(void *node, const lzma_allocator *allocator)) { + assert(free_func != NULL); + if (tree->root != NULL) index_tree_node_end(tree->root, allocator, free_func); @@ -230,7 +229,6 @@ index_tree_end(index_tree *tree, lzma_allocator *allocator, static void index_tree_append(index_tree *tree, index_tree_node *node) { - uint32_t up; node->parent = tree->rightmost; node->left = NULL; node->right = NULL; @@ -259,10 +257,8 @@ index_tree_append(index_tree *tree, index_tree_node *node) // and thus know the state of the tree just by looking at the node // count. From the node count we can calculate how many steps to go // up in the tree to find the rotation root. - up = tree->count ^ (UINT32_C(1) << bsr32(tree->count)); + uint32_t up = tree->count ^ (UINT32_C(1) << bsr32(tree->count)); if (up != 0) { - index_tree_node *pivot; - // Locate the root node for the rotation. up = ctz32(tree->count) + 2; do { @@ -270,7 +266,7 @@ index_tree_append(index_tree *tree, index_tree_node *node) } while (--up > 0); // Rotate left using node as the rotation root. - pivot = node->right; + index_tree_node *pivot = node->right; if (node->parent == NULL) { tree->root = pivot; @@ -342,8 +338,8 @@ index_tree_locate(const index_tree *tree, lzma_vli target) /// Allocate and initialize a new Stream using the given base offsets. static index_stream * index_stream_init(lzma_vli compressed_base, lzma_vli uncompressed_base, - lzma_vli stream_number, lzma_vli block_number_base, - lzma_allocator *allocator) + uint32_t stream_number, lzma_vli block_number_base, + const lzma_allocator *allocator) { index_stream *s = lzma_alloc(sizeof(index_stream), allocator); if (s == NULL) @@ -371,16 +367,17 @@ index_stream_init(lzma_vli compressed_base, lzma_vli uncompressed_base, /// Free the memory allocated for a Stream and its Record groups. static void -index_stream_end(void *node, lzma_allocator *allocator) +index_stream_end(void *node, const lzma_allocator *allocator) { index_stream *s = node; - index_tree_end(&s->groups, allocator, NULL); + index_tree_end(&s->groups, allocator, &lzma_free); + lzma_free(s, allocator); return; } static lzma_index * -index_init_plain(lzma_allocator *allocator) +index_init_plain(const lzma_allocator *allocator) { lzma_index *i = lzma_alloc(sizeof(lzma_index), allocator); if (i != NULL) { @@ -398,15 +395,13 @@ index_init_plain(lzma_allocator *allocator) extern LZMA_API(lzma_index *) -lzma_index_init(lzma_allocator *allocator) +lzma_index_init(const lzma_allocator *allocator) { - index_stream *s; - lzma_index *i = index_init_plain(allocator); if (i == NULL) return NULL; - s = index_stream_init(0, 0, 1, 0, allocator); + index_stream *s = index_stream_init(0, 0, 1, 0, allocator); if (s == NULL) { lzma_free(i, allocator); return NULL; @@ -419,7 +414,7 @@ lzma_index_init(lzma_allocator *allocator) extern LZMA_API(void) -lzma_index_end(lzma_index *i, lzma_allocator *allocator) +lzma_index_end(lzma_index *i, const lzma_allocator *allocator) { // NOTE: If you modify this function, check also the bottom // of lzma_index_cat(). @@ -605,8 +600,6 @@ lzma_index_padding_size(const lzma_index *i) extern LZMA_API(lzma_ret) lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) { - index_stream *s; - if (i == NULL || stream_flags == NULL) return LZMA_PROG_ERROR; @@ -614,7 +607,7 @@ lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) return_if_error(lzma_stream_flags_compare( stream_flags, stream_flags)); - s = (index_stream *)(i->streams.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); s->stream_flags = *stream_flags; return LZMA_OK; @@ -624,17 +617,14 @@ lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) extern LZMA_API(lzma_ret) lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding) { - index_stream *s; - lzma_vli old_stream_padding; - if (i == NULL || stream_padding > LZMA_VLI_MAX || (stream_padding & 3) != 0) return LZMA_PROG_ERROR; - s = (index_stream *)(i->streams.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); // Check that the new value won't make the file grow too big. - old_stream_padding = s->stream_padding; + const lzma_vli old_stream_padding = s->stream_padding; s->stream_padding = 0; if (lzma_index_file_size(i) + stream_padding > LZMA_VLI_MAX) { s->stream_padding = old_stream_padding; @@ -647,29 +637,23 @@ lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding) extern LZMA_API(lzma_ret) -lzma_index_append(lzma_index *i, lzma_allocator *allocator, +lzma_index_append(lzma_index *i, const lzma_allocator *allocator, lzma_vli unpadded_size, lzma_vli uncompressed_size) { - index_stream *s; - index_group *g; - lzma_vli compressed_base; - lzma_vli uncompressed_base; - uint32_t index_list_size_add; - // Validate. if (i == NULL || unpadded_size < UNPADDED_SIZE_MIN || unpadded_size > UNPADDED_SIZE_MAX || uncompressed_size > LZMA_VLI_MAX) return LZMA_PROG_ERROR; - s = (index_stream *)(i->streams.rightmost); - g = (index_group *)(s->groups.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); + index_group *g = (index_group *)(s->groups.rightmost); - compressed_base = g == NULL ? 0 + const lzma_vli compressed_base = g == NULL ? 0 : vli_ceil4(g->records[g->last].unpadded_sum); - uncompressed_base = g == NULL ? 0 + const lzma_vli uncompressed_base = g == NULL ? 0 : g->records[g->last].uncompressed_sum; - index_list_size_add = lzma_vli_size(unpadded_size) + const uint32_t index_list_size_add = lzma_vli_size(unpadded_size) + lzma_vli_size(uncompressed_size); // Check that the file size will stay within limits. @@ -780,10 +764,9 @@ index_cat_helper(const index_cat_info *info, index_stream *this) extern LZMA_API(lzma_ret) -lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, - lzma_allocator *allocator) +lzma_index_cat(lzma_index *restrict dest, lzma_index *restrict src, + const lzma_allocator *allocator) { - index_cat_info info; const lzma_vli dest_file_size = lzma_index_file_size(dest); // Check that we don't exceed the file size limits. @@ -813,12 +796,10 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, index_stream *s = (index_stream *)(dest->streams.rightmost); index_group *g = (index_group *)(s->groups.rightmost); if (g != NULL && g->last + 1 < g->allocated) { - index_group *newg; - assert(g->node.left == NULL); assert(g->node.right == NULL); - newg = lzma_alloc(sizeof(index_group) + index_group *newg = lzma_alloc(sizeof(index_group) + (g->last + 1) * sizeof(index_record), allocator); @@ -848,17 +829,21 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, s->groups.rightmost = &newg->node; lzma_free(g, allocator); + + // NOTE: newg isn't leaked here because + // newg == (void *)&newg->node. } } // Add all the Streams from src to dest. Update the base offsets // of each Stream from src. - info.uncompressed_size = dest->uncompressed_size; - info.file_size = dest_file_size; - info.stream_number_add = dest->streams.count; - info.block_number_add = dest->record_count; - info.streams = &dest->streams; - + const index_cat_info info = { + .uncompressed_size = dest->uncompressed_size, + .file_size = dest_file_size, + .stream_number_add = dest->streams.count, + .block_number_add = dest->record_count, + .streams = &dest->streams, + }; index_cat_helper(&info, (index_stream *)(src->streams.root)); // Update info about all the combined Streams. @@ -877,26 +862,18 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, /// Duplicate an index_stream. static index_stream * -index_dup_stream(const index_stream *src, lzma_allocator *allocator) +index_dup_stream(const index_stream *src, const lzma_allocator *allocator) { - index_stream *dest; - index_group *destg; - index_group *srcg; - size_t i = 0; - // Catch a somewhat theoretical integer overflow. if (src->record_count > PREALLOC_MAX) return NULL; // Allocate and initialize a new Stream. - dest = index_stream_init(src->node.compressed_base, + index_stream *dest = index_stream_init(src->node.compressed_base, src->node.uncompressed_base, src->number, src->block_number_base, allocator); - - // Return immediately if allocation failed or if there are - // no groups to duplicate. - if (dest == NULL || src->groups.leftmost == NULL) - return dest; + if (dest == NULL) + return NULL; // Copy the overall information. dest->record_count = src->record_count; @@ -904,10 +881,14 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) dest->stream_flags = src->stream_flags; dest->stream_padding = src->stream_padding; + // Return if there are no groups to duplicate. + if (src->groups.leftmost == NULL) + return dest; + // Allocate memory for the Records. We put all the Records into // a single group. It's simplest and also tends to make // lzma_index_locate() a little bit faster with very big Indexes. - destg = lzma_alloc(sizeof(index_group) + index_group *destg = lzma_alloc(sizeof(index_group) + src->record_count * sizeof(index_record), allocator); if (destg == NULL) { @@ -923,7 +904,8 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) destg->last = src->record_count - 1; // Go through all the groups in src and copy the Records into destg. - srcg = (index_group *)(src->groups.leftmost); + const index_group *srcg = (const index_group *)(src->groups.leftmost); + size_t i = 0; do { memcpy(destg->records + i, srcg->records, (srcg->last + 1) * sizeof(index_record)); @@ -941,11 +923,8 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) extern LZMA_API(lzma_index *) -lzma_index_dup(const lzma_index *src, lzma_allocator *allocator) +lzma_index_dup(const lzma_index *src, const lzma_allocator *allocator) { - index_stream *srcstream; - index_stream *deststream; - // Allocate the base structure (no initial Stream). lzma_index *dest = index_init_plain(allocator); if (dest == NULL) @@ -958,9 +937,11 @@ lzma_index_dup(const lzma_index *src, lzma_allocator *allocator) dest->index_list_size = src->index_list_size; // Copy the Streams and the groups in them. - srcstream = (index_stream *)(src->streams.leftmost); + const index_stream *srcstream + = (const index_stream *)(src->streams.leftmost); do { - deststream = index_dup_stream(srcstream, allocator); + index_stream *deststream = index_dup_stream( + srcstream, allocator); if (deststream == NULL) { lzma_index_end(dest, allocator); return NULL; @@ -1031,6 +1012,8 @@ iter_set_info(lzma_index_iter *iter) iter->internal[ITER_GROUP].p = NULL; } + // NOTE: lzma_index_iter.stream.number is lzma_vli but we use uint32_t + // internally. iter->stream.number = stream->number; iter->stream.block_count = stream->record_count; iter->stream.compressed_offset = stream->node.compressed_base; @@ -1119,19 +1102,14 @@ lzma_index_iter_rewind(lzma_index_iter *iter) extern LZMA_API(lzma_bool) lzma_index_iter_next(lzma_index_iter *iter, lzma_index_iter_mode mode) { - const lzma_index *i; - const index_stream *stream; - const index_group *group; - size_t record; - // Catch unsupported mode values. if ((unsigned int)(mode) > LZMA_INDEX_ITER_NONEMPTY_BLOCK) return true; - i = iter->internal[ITER_INDEX].p; - stream = iter->internal[ITER_STREAM].p; - group = NULL; - record = iter->internal[ITER_RECORD].s; + const lzma_index *i = iter->internal[ITER_INDEX].p; + const index_stream *stream = iter->internal[ITER_STREAM].p; + const index_group *group = NULL; + size_t record = iter->internal[ITER_RECORD].s; // If we are being asked for the next Stream, leave group to NULL // so that the rest of the this function thinks that this Stream @@ -1231,10 +1209,6 @@ again: extern LZMA_API(lzma_bool) lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) { - const index_stream *stream; - const index_group *group; - size_t left, right; - const lzma_index *i = iter->internal[ITER_INDEX].p; // If the target is past the end of the file, return immediately. @@ -1242,12 +1216,12 @@ lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) return true; // Locate the Stream containing the target offset. - stream = index_tree_locate(&i->streams, target); + const index_stream *stream = index_tree_locate(&i->streams, target); assert(stream != NULL); target -= stream->node.uncompressed_base; // Locate the group containing the target offset. - group = index_tree_locate(&stream->groups, target); + const index_group *group = index_tree_locate(&stream->groups, target); assert(group != NULL); // Use binary search to locate the exact Record. It is the first @@ -1255,8 +1229,8 @@ lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) // This is because we want the rightmost Record that fullfills the // search criterion. It is possible that there are empty Blocks; // we don't want to return them. - left = 0; - right = group->last; + size_t left = 0; + size_t right = group->last; while (left < right) { const size_t pos = left + (right - left) / 2; diff --git a/Utilities/cmliblzma/liblzma/common/index_decoder.c b/Utilities/cmliblzma/liblzma/common/index_decoder.c index 943cfd5..cc07a1b 100644 --- a/Utilities/cmliblzma/liblzma/common/index_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/index_decoder.c @@ -14,7 +14,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INDICATOR, SEQ_COUNT, @@ -50,18 +50,20 @@ struct lzma_coder_s { /// CRC32 of the List of Records field uint32_t crc32; -}; +} lzma_index_coder; static lzma_ret -index_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +index_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, - uint8_t *LZMA_RESTRICT out lzma_attribute((__unused__)), - size_t *LZMA_RESTRICT out_pos lzma_attribute((__unused__)), + uint8_t *restrict out lzma_attribute((__unused__)), + size_t *restrict out_pos lzma_attribute((__unused__)), size_t out_size lzma_attribute((__unused__)), lzma_action action lzma_attribute((__unused__))) { + lzma_index_coder *coder = coder_ptr; + // Similar optimization as in index_encoder.c const size_t in_start = *in_pos; lzma_ret ret = LZMA_OK; @@ -207,8 +209,9 @@ out: static void -index_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +index_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_index_coder *coder = coder_ptr; lzma_index_end(coder->index, allocator); lzma_free(coder, allocator); return; @@ -216,9 +219,11 @@ index_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +index_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_index_coder *coder = coder_ptr; + *memusage = lzma_index_memusage(1, coder->count); *old_memlimit = coder->memlimit; @@ -234,7 +239,7 @@ index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, static lzma_ret -index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, +index_decoder_reset(lzma_index_coder *coder, const lzma_allocator *allocator, lzma_index **i, uint64_t memlimit) { // Remember the pointer given by the application. We will set it @@ -251,7 +256,7 @@ index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, // Initialize the rest. coder->sequence = SEQ_INDICATOR; - coder->memlimit = memlimit; + coder->memlimit = my_max(1, memlimit); coder->count = 0; // Needs to be initialized due to _memconfig(). coder->pos = 0; coder->crc32 = 0; @@ -261,35 +266,37 @@ index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, static lzma_ret -index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_index **i, uint64_t memlimit) { lzma_next_coder_init(&index_decoder_init, next, allocator); - if (i == NULL || memlimit == 0) + if (i == NULL) return LZMA_PROG_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_index_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_index_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &index_decode; next->end = &index_decoder_end; next->memconfig = &index_decoder_memconfig; - next->coder->index = NULL; + coder->index = NULL; } else { - lzma_index_end(next->coder->index, allocator); + lzma_index_end(coder->index, allocator); } - return index_decoder_reset(next->coder, allocator, i, memlimit); + return index_decoder_reset(coder, allocator, i, memlimit); } extern LZMA_API(lzma_ret) lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) { - lzma_next_strm_init2(index_decoder_init, strm, i, memlimit); + lzma_next_strm_init(index_decoder_init, strm, i, memlimit); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -299,27 +306,25 @@ lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) extern LZMA_API(lzma_ret) -lzma_index_buffer_decode( - lzma_index **i, uint64_t *memlimit, lzma_allocator *allocator, +lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { - lzma_coder coder; - lzma_ret ret; - - // Store the input start position so that we can restore it in case - // of an error. - const size_t in_start = *in_pos; - // Sanity checks if (i == NULL || memlimit == NULL || in == NULL || in_pos == NULL || *in_pos > in_size) return LZMA_PROG_ERROR; // Initialize the decoder. + lzma_index_coder coder; return_if_error(index_decoder_reset(&coder, allocator, i, *memlimit)); + // Store the input start position so that we can restore it in case + // of an error. + const size_t in_start = *in_pos; + // Do the actual decoding. - ret = index_decode(&coder, allocator, in, in_pos, in_size, + lzma_ret ret = index_decode(&coder, allocator, in, in_pos, in_size, NULL, NULL, 0, LZMA_RUN); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/index_encoder.c b/Utilities/cmliblzma/liblzma/common/index_encoder.c index 194bf21..ac97d0c 100644 --- a/Utilities/cmliblzma/liblzma/common/index_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/index_encoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INDICATOR, SEQ_COUNT, @@ -37,19 +37,21 @@ struct lzma_coder_s { /// CRC32 of the List of Records field uint32_t crc32; -}; +} lzma_index_coder; static lzma_ret -index_encode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in lzma_attribute((__unused__)), - size_t *LZMA_RESTRICT in_pos lzma_attribute((__unused__)), +index_encode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in lzma_attribute((__unused__)), + size_t *restrict in_pos lzma_attribute((__unused__)), size_t in_size lzma_attribute((__unused__)), - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_action action lzma_attribute((__unused__))) { + lzma_index_coder *coder = coder_ptr; + // Position where to start calculating CRC32. The idea is that we // need to call lzma_crc32() only once per call to index_encode(). const size_t out_start = *out_pos; @@ -159,7 +161,7 @@ out: static void -index_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +index_encoder_end(void *coder, const lzma_allocator *allocator) { lzma_free(coder, allocator); return; @@ -167,7 +169,7 @@ index_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static void -index_encoder_reset(lzma_coder *coder, const lzma_index *i) +index_encoder_reset(lzma_index_coder *coder, const lzma_index *i) { lzma_index_iter_init(&coder->iter, i); @@ -181,7 +183,7 @@ index_encoder_reset(lzma_coder *coder, const lzma_index *i) extern lzma_ret -lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_index_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_index *i) { lzma_next_coder_init(&lzma_index_encoder_init, next, allocator); @@ -190,7 +192,7 @@ lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_PROG_ERROR; if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + next->coder = lzma_alloc(sizeof(lzma_index_coder), allocator); if (next->coder == NULL) return LZMA_MEM_ERROR; @@ -207,7 +209,7 @@ lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_index_encoder(lzma_stream *strm, const lzma_index *i) { - lzma_next_strm_init1(lzma_index_encoder_init, strm, i); + lzma_next_strm_init(lzma_index_encoder_init, strm, i); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -220,10 +222,6 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_coder coder; - size_t out_start; - lzma_ret ret; - // Validate the arguments. if (i == NULL || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; @@ -234,12 +232,13 @@ lzma_index_buffer_encode(const lzma_index *i, // The Index encoder needs just one small data structure so we can // allocate it on stack. + lzma_index_coder coder; index_encoder_reset(&coder, i); // Do the actual encoding. This should never fail, but store // the original *out_pos just in case. - out_start = *out_pos; - ret = index_encode(&coder, NULL, NULL, NULL, 0, + const size_t out_start = *out_pos; + lzma_ret ret = index_encode(&coder, NULL, NULL, NULL, 0, out, out_pos, out_size, LZMA_RUN); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/index_encoder.h b/Utilities/cmliblzma/liblzma/common/index_encoder.h index a13c94d..4d55cd1 100644 --- a/Utilities/cmliblzma/liblzma/common/index_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/index_encoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_index_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_index *i); + const lzma_allocator *allocator, const lzma_index *i); #endif diff --git a/Utilities/cmliblzma/liblzma/common/index_hash.c b/Utilities/cmliblzma/liblzma/common/index_hash.c index 0cf86b3..d7a0344 100644 --- a/Utilities/cmliblzma/liblzma/common/index_hash.c +++ b/Utilities/cmliblzma/liblzma/common/index_hash.c @@ -70,7 +70,8 @@ struct lzma_index_hash_s { extern LZMA_API(lzma_index_hash *) -lzma_index_hash_init(lzma_index_hash *index_hash, lzma_allocator *allocator) +lzma_index_hash_init(lzma_index_hash *index_hash, + const lzma_allocator *allocator) { if (index_hash == NULL) { index_hash = lzma_alloc(sizeof(lzma_index_hash), allocator); @@ -101,7 +102,8 @@ lzma_index_hash_init(lzma_index_hash *index_hash, lzma_allocator *allocator) extern LZMA_API(void) -lzma_index_hash_end(lzma_index_hash *index_hash, lzma_allocator *allocator) +lzma_index_hash_end(lzma_index_hash *index_hash, + const lzma_allocator *allocator) { lzma_free(index_hash, allocator); return; @@ -124,14 +126,13 @@ static lzma_ret hash_append(lzma_index_hash_info *info, lzma_vli unpadded_size, lzma_vli uncompressed_size) { - const lzma_vli sizes[2] = { unpadded_size, uncompressed_size }; - info->blocks_size += vli_ceil4(unpadded_size); info->uncompressed_size += uncompressed_size; info->index_list_size += lzma_vli_size(unpadded_size) + lzma_vli_size(uncompressed_size); ++info->count; + const lzma_vli sizes[2] = { unpadded_size, uncompressed_size }; lzma_check_update(&info->check, LZMA_CHECK_BEST, (const uint8_t *)(sizes), sizeof(sizes)); @@ -174,9 +175,6 @@ extern LZMA_API(lzma_ret) lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, size_t *in_pos, size_t in_size) { - size_t in_start; - lzma_ret ret; - // Catch zero input buffer here, because in contrast to Index encoder // and decoder functions, applications call this function directly // instead of via lzma_code(), which does the buffer checking. @@ -186,8 +184,8 @@ lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, // NOTE: This function has many similarities to index_encode() and // index_decode() functions found from index_encoder.c and // index_decoder.c. See the comments especially in index_encoder.c. - in_start = *in_pos; - ret = LZMA_OK; + const size_t in_start = *in_pos; + lzma_ret ret = LZMA_OK; while (*in_pos < in_size) switch (index_hash->sequence) { diff --git a/Utilities/cmliblzma/liblzma/common/memcmplen.h b/Utilities/cmliblzma/liblzma/common/memcmplen.h new file mode 100644 index 0000000..c1efc9e --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/memcmplen.h @@ -0,0 +1,175 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file memcmplen.h +/// \brief Optimized comparison of two buffers +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_MEMCMPLEN_H +#define LZMA_MEMCMPLEN_H + +#include "common.h" + +#ifdef HAVE_IMMINTRIN_H +# include <immintrin.h> +#endif + + +/// Find out how many equal bytes the two buffers have. +/// +/// \param buf1 First buffer +/// \param buf2 Second buffer +/// \param len How many bytes have already been compared and will +/// be assumed to match +/// \param limit How many bytes to compare at most, including the +/// already-compared bytes. This must be significantly +/// smaller than UINT32_MAX to avoid integer overflows. +/// Up to LZMA_MEMCMPLEN_EXTRA bytes may be read past +/// the specified limit from both buf1 and buf2. +/// +/// \return Number of equal bytes in the buffers is returned. +/// This is always at least len and at most limit. +/// +/// \note LZMA_MEMCMPLEN_EXTRA defines how many extra bytes may be read. +/// It's rounded up to 2^n. This extra amount needs to be +/// allocated in the buffers being used. It needs to be +/// initialized too to keep Valgrind quiet. +static inline uint32_t lzma_attribute((__always_inline__)) +lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2, + uint32_t len, uint32_t limit) +{ + assert(len <= limit); + assert(limit <= UINT32_MAX / 2); + +#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ + && ((TUKLIB_GNUC_REQ(3, 4) && defined(__x86_64__)) \ + || (defined(__INTEL_COMPILER) && defined(__x86_64__)) \ + || (defined(__INTEL_COMPILER) && defined(_M_X64)) \ + || (defined(_MSC_VER) && defined(_M_X64))) + // NOTE: This will use 64-bit unaligned access which + // TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit, but + // it's convenient here at least as long as it's x86-64 only. + // + // I keep this x86-64 only for now since that's where I know this + // to be a good method. This may be fine on other 64-bit CPUs too. + // On big endian one should use xor instead of subtraction and switch + // to __builtin_clzll(). +#define LZMA_MEMCMPLEN_EXTRA 8 + while (len < limit) { + const uint64_t x = *(const uint64_t *)(buf1 + len) + - *(const uint64_t *)(buf2 + len); + if (x != 0) { +# if defined(_M_X64) // MSVC or Intel C compiler on Windows + unsigned long tmp; + _BitScanForward64(&tmp, x); + len += (uint32_t)tmp >> 3; +# else // GCC, clang, or Intel C compiler + len += (uint32_t)__builtin_ctzll(x) >> 3; +# endif + return my_min(len, limit); + } + + len += 8; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ + && defined(HAVE__MM_MOVEMASK_EPI8) \ + && ((defined(__GNUC__) && defined(__SSE2_MATH__)) \ + || (defined(__INTEL_COMPILER) && defined(__SSE2__)) \ + || (defined(_MSC_VER) && defined(_M_IX86_FP) \ + && _M_IX86_FP >= 2)) + // NOTE: Like above, this will use 128-bit unaligned access which + // TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit. + // + // SSE2 version for 32-bit and 64-bit x86. On x86-64 the above + // version is sometimes significantly faster and sometimes + // slightly slower than this SSE2 version, so this SSE2 + // version isn't used on x86-64. +# define LZMA_MEMCMPLEN_EXTRA 16 + while (len < limit) { + const uint32_t x = 0xFFFF ^ _mm_movemask_epi8(_mm_cmpeq_epi8( + _mm_loadu_si128((const __m128i *)(buf1 + len)), + _mm_loadu_si128((const __m128i *)(buf2 + len)))); + + if (x != 0) { +# if defined(__INTEL_COMPILER) + len += _bit_scan_forward(x); +# elif defined(_MSC_VER) + unsigned long tmp; + _BitScanForward(&tmp, x); + len += tmp; +# else + len += __builtin_ctz(x); +# endif + return my_min(len, limit); + } + + len += 16; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && !defined(WORDS_BIGENDIAN) + // Generic 32-bit little endian method +# define LZMA_MEMCMPLEN_EXTRA 4 + while (len < limit) { + uint32_t x = *(const uint32_t *)(buf1 + len) + - *(const uint32_t *)(buf2 + len); + if (x != 0) { + if ((x & 0xFFFF) == 0) { + len += 2; + x >>= 16; + } + + if ((x & 0xFF) == 0) + ++len; + + return my_min(len, limit); + } + + len += 4; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && defined(WORDS_BIGENDIAN) + // Generic 32-bit big endian method +# define LZMA_MEMCMPLEN_EXTRA 4 + while (len < limit) { + uint32_t x = *(const uint32_t *)(buf1 + len) + ^ *(const uint32_t *)(buf2 + len); + if (x != 0) { + if ((x & 0xFFFF0000) == 0) { + len += 2; + x <<= 16; + } + + if ((x & 0xFF000000) == 0) + ++len; + + return my_min(len, limit); + } + + len += 4; + } + + return limit; + +#else + // Simple portable version that doesn't use unaligned access. +# define LZMA_MEMCMPLEN_EXTRA 0 + while (len < limit && buf1[len] == buf2[len]) + ++len; + + return len; +#endif +} + +#endif diff --git a/Utilities/cmliblzma/liblzma/common/outqueue.c b/Utilities/cmliblzma/liblzma/common/outqueue.c new file mode 100644 index 0000000..2dc8a38 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/outqueue.c @@ -0,0 +1,184 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file outqueue.c +/// \brief Output queue handling in multithreaded coding +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "outqueue.h" + + +/// This is to ease integer overflow checking: We may allocate up to +/// 2 * LZMA_THREADS_MAX buffers and we need some extra memory for other +/// data structures (that's the second /2). +#define BUF_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2) + + +static lzma_ret +get_options(uint64_t *bufs_alloc_size, uint32_t *bufs_count, + uint64_t buf_size_max, uint32_t threads) +{ + if (threads > LZMA_THREADS_MAX || buf_size_max > BUF_SIZE_MAX) + return LZMA_OPTIONS_ERROR; + + // The number of buffers is twice the number of threads. + // This wastes RAM but keeps the threads busy when buffers + // finish out of order. + // + // NOTE: If this is changed, update BUF_SIZE_MAX too. + *bufs_count = threads * 2; + *bufs_alloc_size = *bufs_count * buf_size_max; + + return LZMA_OK; +} + + +extern uint64_t +lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads) +{ + uint64_t bufs_alloc_size; + uint32_t bufs_count; + + if (get_options(&bufs_alloc_size, &bufs_count, buf_size_max, threads) + != LZMA_OK) + return UINT64_MAX; + + return sizeof(lzma_outq) + bufs_count * sizeof(lzma_outbuf) + + bufs_alloc_size; +} + + +extern lzma_ret +lzma_outq_init(lzma_outq *outq, const lzma_allocator *allocator, + uint64_t buf_size_max, uint32_t threads) +{ + uint64_t bufs_alloc_size; + uint32_t bufs_count; + + // Set bufs_count and bufs_alloc_size. + return_if_error(get_options(&bufs_alloc_size, &bufs_count, + buf_size_max, threads)); + + // Allocate memory if needed. + if (outq->buf_size_max != buf_size_max + || outq->bufs_allocated != bufs_count) { + lzma_outq_end(outq, allocator); + +#if SIZE_MAX < UINT64_MAX + if (bufs_alloc_size > SIZE_MAX) + return LZMA_MEM_ERROR; +#endif + + outq->bufs = lzma_alloc(bufs_count * sizeof(lzma_outbuf), + allocator); + outq->bufs_mem = lzma_alloc((size_t)(bufs_alloc_size), + allocator); + + if (outq->bufs == NULL || outq->bufs_mem == NULL) { + lzma_outq_end(outq, allocator); + return LZMA_MEM_ERROR; + } + } + + // Initialize the rest of the main structure. Initialization of + // outq->bufs[] is done when they are actually needed. + outq->buf_size_max = (size_t)(buf_size_max); + outq->bufs_allocated = bufs_count; + outq->bufs_pos = 0; + outq->bufs_used = 0; + outq->read_pos = 0; + + return LZMA_OK; +} + + +extern void +lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator) +{ + lzma_free(outq->bufs, allocator); + outq->bufs = NULL; + + lzma_free(outq->bufs_mem, allocator); + outq->bufs_mem = NULL; + + return; +} + + +extern lzma_outbuf * +lzma_outq_get_buf(lzma_outq *outq) +{ + // Caller must have checked it with lzma_outq_has_buf(). + assert(outq->bufs_used < outq->bufs_allocated); + + // Initialize the new buffer. + lzma_outbuf *buf = &outq->bufs[outq->bufs_pos]; + buf->buf = outq->bufs_mem + outq->bufs_pos * outq->buf_size_max; + buf->size = 0; + buf->finished = false; + + // Update the queue state. + if (++outq->bufs_pos == outq->bufs_allocated) + outq->bufs_pos = 0; + + ++outq->bufs_used; + + return buf; +} + + +extern bool +lzma_outq_is_readable(const lzma_outq *outq) +{ + uint32_t i = outq->bufs_pos - outq->bufs_used; + if (outq->bufs_pos < outq->bufs_used) + i += outq->bufs_allocated; + + return outq->bufs[i].finished; +} + + +extern lzma_ret +lzma_outq_read(lzma_outq *restrict outq, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_vli *restrict unpadded_size, + lzma_vli *restrict uncompressed_size) +{ + // There must be at least one buffer from which to read. + if (outq->bufs_used == 0) + return LZMA_OK; + + // Get the buffer. + uint32_t i = outq->bufs_pos - outq->bufs_used; + if (outq->bufs_pos < outq->bufs_used) + i += outq->bufs_allocated; + + lzma_outbuf *buf = &outq->bufs[i]; + + // If it isn't finished yet, we cannot read from it. + if (!buf->finished) + return LZMA_OK; + + // Copy from the buffer to output. + lzma_bufcpy(buf->buf, &outq->read_pos, buf->size, + out, out_pos, out_size); + + // Return if we didn't get all the data from the buffer. + if (outq->read_pos < buf->size) + return LZMA_OK; + + // The buffer was finished. Tell the caller its size information. + *unpadded_size = buf->unpadded_size; + *uncompressed_size = buf->uncompressed_size; + + // Free this buffer for further use. + --outq->bufs_used; + outq->read_pos = 0; + + return LZMA_STREAM_END; +} diff --git a/Utilities/cmliblzma/liblzma/common/outqueue.h b/Utilities/cmliblzma/liblzma/common/outqueue.h new file mode 100644 index 0000000..079634d --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/outqueue.h @@ -0,0 +1,156 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file outqueue.h +/// \brief Output queue handling in multithreaded coding +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +/// Output buffer for a single thread +typedef struct { + /// Pointer to the output buffer of lzma_outq.buf_size_max bytes + uint8_t *buf; + + /// Amount of data written to buf + size_t size; + + /// Additional size information + lzma_vli unpadded_size; + lzma_vli uncompressed_size; + + /// True when no more data will be written into this buffer. + /// + /// \note This is read by another thread and thus access + /// to this variable needs a mutex. + bool finished; + +} lzma_outbuf; + + +typedef struct { + /// Array of buffers that are used cyclically. + lzma_outbuf *bufs; + + /// Memory allocated for all the buffers + uint8_t *bufs_mem; + + /// Amount of buffer space available in each buffer + size_t buf_size_max; + + /// Number of buffers allocated + uint32_t bufs_allocated; + + /// Position in the bufs array. The next buffer to be taken + /// into use is bufs[bufs_pos]. + uint32_t bufs_pos; + + /// Number of buffers in use + uint32_t bufs_used; + + /// Position in the buffer in lzma_outq_read() + size_t read_pos; + +} lzma_outq; + + +/** + * \brief Calculate the memory usage of an output queue + * + * \return Approximate memory usage in bytes or UINT64_MAX on error. + */ +extern uint64_t lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads); + + +/// \brief Initialize an output queue +/// +/// \param outq Pointer to an output queue. Before calling +/// this function the first time, *outq should +/// have been zeroed with memzero() so that this +/// function knows that there are no previous +/// allocations to free. +/// \param allocator Pointer to allocator or NULL +/// \param buf_size_max Maximum amount of data that a single buffer +/// in the queue may need to store. +/// \param threads Number of buffers that may be in use +/// concurrently. Note that more than this number +/// of buffers will actually get allocated to +/// improve performance when buffers finish +/// out of order. +/// +/// \return - LZMA_OK +/// - LZMA_MEM_ERROR +/// +extern lzma_ret lzma_outq_init( + lzma_outq *outq, const lzma_allocator *allocator, + uint64_t buf_size_max, uint32_t threads); + + +/// \brief Free the memory associated with the output queue +extern void lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator); + + +/// \brief Get a new buffer +/// +/// lzma_outq_has_buf() must be used to check that there is a buffer +/// available before calling lzma_outq_get_buf(). +/// +extern lzma_outbuf *lzma_outq_get_buf(lzma_outq *outq); + + +/// \brief Test if there is data ready to be read +/// +/// Call to this function must be protected with the same mutex that +/// is used to protect lzma_outbuf.finished. +/// +extern bool lzma_outq_is_readable(const lzma_outq *outq); + + +/// \brief Read finished data +/// +/// \param outq Pointer to an output queue +/// \param out Beginning of the output buffer +/// \param out_pos The next byte will be written to +/// out[*out_pos]. +/// \param out_size Size of the out buffer; the first byte into +/// which no data is written to is out[out_size]. +/// \param unpadded_size Unpadded Size from the Block encoder +/// \param uncompressed_size Uncompressed Size from the Block encoder +/// +/// \return - LZMA: All OK. Either no data was available or the buffer +/// being read didn't become empty yet. +/// - LZMA_STREAM_END: The buffer being read was finished. +/// *unpadded_size and *uncompressed_size were set. +/// +/// \note This reads lzma_outbuf.finished variables and thus call +/// to this function needs to be protected with a mutex. +/// +extern lzma_ret lzma_outq_read(lzma_outq *restrict outq, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, lzma_vli *restrict unpadded_size, + lzma_vli *restrict uncompressed_size); + + +/// \brief Test if there is at least one buffer free +/// +/// This must be used before getting a new buffer with lzma_outq_get_buf(). +/// +static inline bool +lzma_outq_has_buf(const lzma_outq *outq) +{ + return outq->bufs_used < outq->bufs_allocated; +} + + +/// \brief Test if the queue is completely empty +static inline bool +lzma_outq_is_empty(const lzma_outq *outq) +{ + return outq->bufs_used == 0; +} diff --git a/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c index 9e2e1da..b9745b5 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c @@ -15,13 +15,10 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags, - lzma_allocator *allocator, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder stream_decoder = LZMA_NEXT_CODER_INIT; - lzma_ret ret; - // Sanity checks if (in_pos == NULL || (in == NULL && *in_pos != in_size) || *in_pos > in_size || out_pos == NULL @@ -36,7 +33,8 @@ lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags, // Initialize the Stream decoder. // TODO: We need something to tell the decoder that it can use the // output buffer as workspace, and thus save significant amount of RAM. - ret = lzma_stream_decoder_init( + lzma_next_coder stream_decoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_stream_decoder_init( &stream_decoder, allocator, *memlimit, flags); if (ret == LZMA_OK) { diff --git a/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c index 8bca87f..af49554 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c @@ -42,13 +42,10 @@ lzma_stream_buffer_bound(size_t uncompressed_size) extern LZMA_API(lzma_ret) lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos_ptr, size_t out_size) { - lzma_stream_flags stream_flags = { 0 }; - lzma_block block = { 0 }; - size_t out_pos; - // Sanity checks if (filters == NULL || (unsigned int)(check) > LZMA_CHECK_ID_MAX || (in == NULL && in_size != 0) || out == NULL @@ -65,7 +62,7 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, // Use a local copy. We update *out_pos_ptr only if everything // succeeds. - out_pos = *out_pos_ptr; + size_t out_pos = *out_pos_ptr; // Check that there's enough space for both Stream Header and // Stream Footer. @@ -77,7 +74,10 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, out_size -= LZMA_STREAM_HEADER_SIZE; // Encode the Stream Header. - stream_flags.check = check; + lzma_stream_flags stream_flags = { + .version = 0, + .check = check, + }; if (lzma_stream_header_encode(&stream_flags, out + out_pos) != LZMA_OK) @@ -86,8 +86,11 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, out_pos += LZMA_STREAM_HEADER_SIZE; // Encode a Block but only if there is at least one byte of input. - block.check = check; - block.filters = filters; + lzma_block block = { + .version = 0, + .check = check, + .filters = filters, + }; if (in_size > 0) return_if_error(lzma_block_buffer_encode(&block, allocator, @@ -95,8 +98,6 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, // Index { - lzma_ret ret; - // Create an Index. It will have one Record if there was // at least one byte of input to encode. Otherwise the // Index will be empty. @@ -104,7 +105,7 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, if (i == NULL) return LZMA_MEM_ERROR; - ret = LZMA_OK; + lzma_ret ret = LZMA_OK; if (in_size > 0) ret = lzma_index_append(i, allocator, diff --git a/Utilities/cmliblzma/liblzma/common/stream_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_decoder.c index 5e9a220..fdd8ff2 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_decoder.c @@ -14,7 +14,7 @@ #include "block_decoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_STREAM_HEADER, SEQ_BLOCK_HEADER, @@ -57,6 +57,10 @@ struct lzma_coder_s { /// If true, LZMA_GET_CHECK is returned after decoding Stream Header. bool tell_any_check; + /// If true, we will tell the Block decoder to skip calculating + /// and verifying the integrity check. + bool ignore_check; + /// If true, we will decode concatenated Streams that possibly have /// Stream Padding between or after them. LZMA_STREAM_END is returned /// once the application isn't giving us any new input, and we aren't @@ -76,11 +80,11 @@ struct lzma_coder_s { /// Buffer to hold Stream Header, Block Header, and Stream Footer. /// Block Header has biggest maximum size. uint8_t buffer[LZMA_BLOCK_HEADER_SIZE_MAX]; -}; +} lzma_stream_coder; static lzma_ret -stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator) +stream_decoder_reset(lzma_stream_coder *coder, const lzma_allocator *allocator) { // Initialize the Index hash used to verify the Index. coder->index_hash = lzma_index_hash_init(coder->index_hash, allocator); @@ -96,18 +100,18 @@ stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +stream_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_stream_coder *coder = coder_ptr; + // When decoding the actual Block, it may be able to produce more // output even if we don't give it any new input. while (true) switch (coder->sequence) { case SEQ_STREAM_HEADER: { - lzma_ret ret; - // Copy the Stream Header to the internal buffer. lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); @@ -119,7 +123,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, coder->pos = 0; // Decode the Stream Header. - ret = lzma_stream_header_decode( + const lzma_ret ret = lzma_stream_header_decode( &coder->stream_flags, coder->buffer); if (ret != LZMA_OK) return ret == LZMA_FORMAT_ERROR && !coder->first_stream @@ -156,11 +160,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_BLOCK_HEADER: { - lzma_filter filters[LZMA_FILTERS_MAX + 1]; - uint64_t memusage; - lzma_ret ret; - size_t i; - if (*in_pos >= in_size) return LZMA_OK; @@ -189,20 +188,28 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, coder->pos = 0; - // Version 0 is currently the only possible version. - coder->block_options.version = 0; + // Version 1 is needed to support the .ignore_check option. + coder->block_options.version = 1; // Set up a buffer to hold the filter chain. Block Header // decoder will initialize all members of this array so // we don't need to do it here. + lzma_filter filters[LZMA_FILTERS_MAX + 1]; coder->block_options.filters = filters; // Decode the Block Header. return_if_error(lzma_block_header_decode(&coder->block_options, allocator, coder->buffer)); + // If LZMA_IGNORE_CHECK was used, this flag needs to be set. + // It has to be set after lzma_block_header_decode() because + // it always resets this to false. + coder->block_options.ignore_check = coder->ignore_check; + // Check the memory usage limit. - memusage = lzma_raw_decoder_memusage(filters); + const uint64_t memusage = lzma_raw_decoder_memusage(filters); + lzma_ret ret; + if (memusage == UINT64_MAX) { // One or more unknown Filter IDs. ret = LZMA_OPTIONS_ERROR; @@ -228,7 +235,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Free the allocated filter options since they are needed // only to initialize the Block decoder. - for (i = 0; i < LZMA_FILTERS_MAX; ++i) + for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) lzma_free(filters[i].options, allocator); coder->block_options.filters = NULL; @@ -264,8 +271,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_INDEX: { - lzma_ret ret; - // If we don't have any input, don't call // lzma_index_hash_decode() since it would return // LZMA_BUF_ERROR, which we must not do here. @@ -274,7 +279,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Decode the Index and compare it to the hash calculated // from the sizes of the Blocks (if any). - ret = lzma_index_hash_decode(coder->index_hash, + const lzma_ret ret = lzma_index_hash_decode(coder->index_hash, in, in_pos, in_size); if (ret != LZMA_STREAM_END) return ret; @@ -285,9 +290,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_STREAM_FOOTER: { - lzma_stream_flags footer_flags; - lzma_ret ret; - // Copy the Stream Footer to the internal buffer. lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); @@ -301,7 +303,8 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Decode the Stream Footer. The decoder gives // LZMA_FORMAT_ERROR if the magic bytes don't match, // so convert that return code to LZMA_DATA_ERROR. - ret = lzma_stream_footer_decode( + lzma_stream_flags footer_flags; + const lzma_ret ret = lzma_stream_footer_decode( &footer_flags, coder->buffer); if (ret != LZMA_OK) return ret == LZMA_FORMAT_ERROR @@ -374,8 +377,9 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, static void -stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +stream_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_stream_coder *coder = coder_ptr; lzma_next_end(&coder->block_decoder, allocator); lzma_index_hash_end(coder->index_hash, allocator); lzma_free(coder, allocator); @@ -384,16 +388,19 @@ stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_check -stream_decoder_get_check(const lzma_coder *coder) +stream_decoder_get_check(const void *coder_ptr) { + const lzma_stream_coder *coder = coder_ptr; return coder->stream_flags.check; } static lzma_ret -stream_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +stream_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_stream_coder *coder = coder_ptr; + *memusage = coder->memusage; *old_memlimit = coder->memlimit; @@ -409,48 +416,49 @@ stream_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, extern lzma_ret -lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_stream_decoder_init( + lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, uint32_t flags) { lzma_next_coder_init(&lzma_stream_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; - if (flags & ~LZMA_SUPPORTED_FLAGS) return LZMA_OPTIONS_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_stream_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &stream_decode; next->end = &stream_decoder_end; next->get_check = &stream_decoder_get_check; next->memconfig = &stream_decoder_memconfig; - next->coder->block_decoder = LZMA_NEXT_CODER_INIT; - next->coder->index_hash = NULL; + coder->block_decoder = LZMA_NEXT_CODER_INIT; + coder->index_hash = NULL; } - next->coder->memlimit = memlimit; - next->coder->memusage = LZMA_MEMUSAGE_BASE; - next->coder->tell_no_check = (flags & LZMA_TELL_NO_CHECK) != 0; - next->coder->tell_unsupported_check + coder->memlimit = my_max(1, memlimit); + coder->memusage = LZMA_MEMUSAGE_BASE; + coder->tell_no_check = (flags & LZMA_TELL_NO_CHECK) != 0; + coder->tell_unsupported_check = (flags & LZMA_TELL_UNSUPPORTED_CHECK) != 0; - next->coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0; - next->coder->concatenated = (flags & LZMA_CONCATENATED) != 0; - next->coder->first_stream = true; + coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0; + coder->ignore_check = (flags & LZMA_IGNORE_CHECK) != 0; + coder->concatenated = (flags & LZMA_CONCATENATED) != 0; + coder->first_stream = true; - return stream_decoder_reset(next->coder, allocator); + return stream_decoder_reset(coder, allocator); } extern LZMA_API(lzma_ret) lzma_stream_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init2(lzma_stream_decoder_init, strm, memlimit, flags); + lzma_next_strm_init(lzma_stream_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/stream_decoder.h b/Utilities/cmliblzma/liblzma/common/stream_decoder.h index e54ac28..c13c6ba 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/stream_decoder.h @@ -15,7 +15,8 @@ #include "common.h" -extern lzma_ret lzma_stream_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, uint64_t memlimit, uint32_t flags); +extern lzma_ret lzma_stream_decoder_init( + lzma_next_coder *next, const lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags); #endif diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_encoder.c index 1ba45ac..858cba4 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_encoder.c @@ -10,12 +10,11 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_encoder.h" #include "block_encoder.h" #include "index_encoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_STREAM_HEADER, SEQ_BLOCK_INIT, @@ -26,7 +25,7 @@ struct lzma_coder_s { } sequence; /// True if Block encoder has been initialized by - /// lzma_stream_encoder_init() or stream_encoder_update() + /// stream_encoder_init() or stream_encoder_update() /// and thus doesn't need to be initialized in stream_encode(). bool block_encoder_is_initialized; @@ -56,11 +55,11 @@ struct lzma_coder_s { /// Buffer to hold Stream Header, Block Header, and Stream Footer. /// Block Header has biggest maximum size. uint8_t buffer[LZMA_BLOCK_HEADER_SIZE_MAX]; -}; +} lzma_stream_coder; static lzma_ret -block_encoder_init(lzma_coder *coder, lzma_allocator *allocator) +block_encoder_init(lzma_stream_coder *coder, const lzma_allocator *allocator) { // Prepare the Block options. Even though Block encoder doesn't need // compressed_size, uncompressed_size, and header_size to be @@ -79,11 +78,13 @@ block_encoder_init(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +stream_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_stream_coder *coder = coder_ptr; + // Main loop while (*out_pos < out_size) switch (coder->sequence) { @@ -126,7 +127,7 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } // Initialize the Block encoder unless it was already - // initialized by lzma_stream_encoder_init() or + // initialized by stream_encoder_init() or // stream_encoder_update(). if (!coder->block_encoder_is_initialized) return_if_error(block_encoder_init(coder, allocator)); @@ -147,13 +148,12 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_BLOCK_ENCODE: { - lzma_vli unpadded_size; - - static const lzma_action convert[4] = { + static const lzma_action convert[LZMA_ACTION_MAX + 1] = { LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FINISH, LZMA_FINISH, + LZMA_FINISH, }; const lzma_ret ret = coder->block_encoder.code( @@ -164,7 +164,7 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, return ret; // Add a new Index Record. - unpadded_size = lzma_block_unpadded_size( + const lzma_vli unpadded_size = lzma_block_unpadded_size( &coder->block_options); assert(unpadded_size != 0); return_if_error(lzma_index_append(coder->index, allocator, @@ -176,12 +176,6 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_INDEX_ENCODE: { - const lzma_stream_flags stream_flags = { - 0, - lzma_index_size(coder->index), - coder->block_options.check, - }; - // Call the Index encoder. It doesn't take any input, so // those pointers can be NULL. const lzma_ret ret = coder->index_encoder.code( @@ -192,6 +186,11 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, return ret; // Encode the Stream Footer into coder->buffer. + const lzma_stream_flags stream_flags = { + .version = 0, + .backward_size = lzma_index_size(coder->index), + .check = coder->block_options.check, + }; if (lzma_stream_footer_encode(&stream_flags, coder->buffer) != LZMA_OK) @@ -212,15 +211,15 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, static void -stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +stream_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { - size_t i; + lzma_stream_coder *coder = coder_ptr; lzma_next_end(&coder->block_encoder, allocator); lzma_next_end(&coder->index_encoder, allocator); lzma_index_end(coder->index, allocator); - for (i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) lzma_free(coder->filters[i].options, allocator); lzma_free(coder, allocator); @@ -229,22 +228,20 @@ stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +stream_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters, const lzma_filter *reversed_filters) { - size_t i; + lzma_stream_coder *coder = coder_ptr; if (coder->sequence <= SEQ_BLOCK_INIT) { - lzma_ret ret; - // There is no incomplete Block waiting to be finished, // thus we can change the whole filter chain. Start by // trying to initialize the Block encoder with the new // chain. This way we detect if the chain is valid. coder->block_encoder_is_initialized = false; coder->block_options.filters = (lzma_filter *)(filters); - ret = block_encoder_init(coder, allocator); + const lzma_ret ret = block_encoder_init(coder, allocator); coder->block_options.filters = coder->filters; if (ret != LZMA_OK) return ret; @@ -264,62 +261,66 @@ stream_encoder_update(lzma_coder *coder, lzma_allocator *allocator, } // Free the copy of the old chain and make a copy of the new chain. - for (i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) lzma_free(coder->filters[i].options, allocator); return lzma_filters_copy(filters, coder->filters, allocator); } -extern lzma_ret -lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +static lzma_ret +stream_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters, lzma_check check) { - lzma_stream_flags stream_flags = { 0, 0, check }; - - lzma_next_coder_init(&lzma_stream_encoder_init, next, allocator); + lzma_next_coder_init(&stream_encoder_init, next, allocator); if (filters == NULL) return LZMA_PROG_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_stream_coder *coder = next->coder; + + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &stream_encode; next->end = &stream_encoder_end; next->update = &stream_encoder_update; - next->coder->filters[0].id = LZMA_VLI_UNKNOWN; - next->coder->block_encoder = LZMA_NEXT_CODER_INIT; - next->coder->index_encoder = LZMA_NEXT_CODER_INIT; - next->coder->index = NULL; + coder->filters[0].id = LZMA_VLI_UNKNOWN; + coder->block_encoder = LZMA_NEXT_CODER_INIT; + coder->index_encoder = LZMA_NEXT_CODER_INIT; + coder->index = NULL; } // Basic initializations - next->coder->sequence = SEQ_STREAM_HEADER; - next->coder->block_options.version = 0; - next->coder->block_options.check = check; + coder->sequence = SEQ_STREAM_HEADER; + coder->block_options.version = 0; + coder->block_options.check = check; // Initialize the Index - lzma_index_end(next->coder->index, allocator); - next->coder->index = lzma_index_init(allocator); - if (next->coder->index == NULL) + lzma_index_end(coder->index, allocator); + coder->index = lzma_index_init(allocator); + if (coder->index == NULL) return LZMA_MEM_ERROR; // Encode the Stream Header + lzma_stream_flags stream_flags = { + .version = 0, + .check = check, + }; return_if_error(lzma_stream_header_encode( - &stream_flags, next->coder->buffer)); + &stream_flags, coder->buffer)); - next->coder->buffer_pos = 0; - next->coder->buffer_size = LZMA_STREAM_HEADER_SIZE; + coder->buffer_pos = 0; + coder->buffer_size = LZMA_STREAM_HEADER_SIZE; // Initialize the Block encoder. This way we detect unsupported // filter chains when initializing the Stream encoder instead of // giving an error after Stream Header has already written out. - return stream_encoder_update( - next->coder, allocator, filters, NULL); + return stream_encoder_update(coder, allocator, filters, NULL); } @@ -327,11 +328,12 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder(lzma_stream *strm, const lzma_filter *filters, lzma_check check) { - lzma_next_strm_init2(lzma_stream_encoder_init, strm, filters, check); + lzma_next_strm_init(stream_encoder_init, strm, filters, check); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; strm->internal->supported_actions[LZMA_FULL_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_BARRIER] = true; strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c b/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c new file mode 100644 index 0000000..2efe44c --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c @@ -0,0 +1,1143 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_encoder_mt.c +/// \brief Multithreaded .xz Stream encoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_encoder.h" +#include "easy_preset.h" +#include "block_encoder.h" +#include "block_buffer_encoder.h" +#include "index_encoder.h" +#include "outqueue.h" + + +/// Maximum supported block size. This makes it simpler to prevent integer +/// overflows if we are given unusually large block size. +#define BLOCK_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX) + + +typedef enum { + /// Waiting for work. + THR_IDLE, + + /// Encoding is in progress. + THR_RUN, + + /// Encoding is in progress but no more input data will + /// be read. + THR_FINISH, + + /// The main thread wants the thread to stop whatever it was doing + /// but not exit. + THR_STOP, + + /// The main thread wants the thread to exit. We could use + /// cancellation but since there's stopped anyway, this is lazier. + THR_EXIT, + +} worker_state; + +typedef struct lzma_stream_coder_s lzma_stream_coder; + +typedef struct worker_thread_s worker_thread; +struct worker_thread_s { + worker_state state; + + /// Input buffer of coder->block_size bytes. The main thread will + /// put new input into this and update in_size accordingly. Once + /// no more input is coming, state will be set to THR_FINISH. + uint8_t *in; + + /// Amount of data available in the input buffer. This is modified + /// only by the main thread. + size_t in_size; + + /// Output buffer for this thread. This is set by the main + /// thread every time a new Block is started with this thread + /// structure. + lzma_outbuf *outbuf; + + /// Pointer to the main structure is needed when putting this + /// thread back to the stack of free threads. + lzma_stream_coder *coder; + + /// The allocator is set by the main thread. Since a copy of the + /// pointer is kept here, the application must not change the + /// allocator before calling lzma_end(). + const lzma_allocator *allocator; + + /// Amount of uncompressed data that has already been compressed. + uint64_t progress_in; + + /// Amount of compressed data that is ready. + uint64_t progress_out; + + /// Block encoder + lzma_next_coder block_encoder; + + /// Compression options for this Block + lzma_block block_options; + + /// Next structure in the stack of free worker threads. + worker_thread *next; + + mythread_mutex mutex; + mythread_cond cond; + + /// The ID of this thread is used to join the thread + /// when it's not needed anymore. + mythread thread_id; +}; + + +struct lzma_stream_coder_s { + enum { + SEQ_STREAM_HEADER, + SEQ_BLOCK, + SEQ_INDEX, + SEQ_STREAM_FOOTER, + } sequence; + + /// Start a new Block every block_size bytes of input unless + /// LZMA_FULL_FLUSH or LZMA_FULL_BARRIER is used earlier. + size_t block_size; + + /// The filter chain currently in use + lzma_filter filters[LZMA_FILTERS_MAX + 1]; + + + /// Index to hold sizes of the Blocks + lzma_index *index; + + /// Index encoder + lzma_next_coder index_encoder; + + + /// Stream Flags for encoding the Stream Header and Stream Footer. + lzma_stream_flags stream_flags; + + /// Buffer to hold Stream Header and Stream Footer. + uint8_t header[LZMA_STREAM_HEADER_SIZE]; + + /// Read position in header[] + size_t header_pos; + + + /// Output buffer queue for compressed data + lzma_outq outq; + + + /// Maximum wait time if cannot use all the input and cannot + /// fill the output buffer. This is in milliseconds. + uint32_t timeout; + + + /// Error code from a worker thread + lzma_ret thread_error; + + /// Array of allocated thread-specific structures + worker_thread *threads; + + /// Number of structures in "threads" above. This is also the + /// number of threads that will be created at maximum. + uint32_t threads_max; + + /// Number of thread structures that have been initialized, and + /// thus the number of worker threads actually created so far. + uint32_t threads_initialized; + + /// Stack of free threads. When a thread finishes, it puts itself + /// back into this stack. This starts as empty because threads + /// are created only when actually needed. + worker_thread *threads_free; + + /// The most recent worker thread to which the main thread writes + /// the new input from the application. + worker_thread *thr; + + + /// Amount of uncompressed data in Blocks that have already + /// been finished. + uint64_t progress_in; + + /// Amount of compressed data in Stream Header + Blocks that + /// have already been finished. + uint64_t progress_out; + + + mythread_mutex mutex; + mythread_cond cond; +}; + + +/// Tell the main thread that something has gone wrong. +static void +worker_error(worker_thread *thr, lzma_ret ret) +{ + assert(ret != LZMA_OK); + assert(ret != LZMA_STREAM_END); + + mythread_sync(thr->coder->mutex) { + if (thr->coder->thread_error == LZMA_OK) + thr->coder->thread_error = ret; + + mythread_cond_signal(&thr->coder->cond); + } + + return; +} + + +static worker_state +worker_encode(worker_thread *thr, worker_state state) +{ + assert(thr->progress_in == 0); + assert(thr->progress_out == 0); + + // Set the Block options. + thr->block_options = (lzma_block){ + .version = 0, + .check = thr->coder->stream_flags.check, + .compressed_size = thr->coder->outq.buf_size_max, + .uncompressed_size = thr->coder->block_size, + + // TODO: To allow changing the filter chain, the filters + // array must be copied to each worker_thread. + .filters = thr->coder->filters, + }; + + // Calculate maximum size of the Block Header. This amount is + // reserved in the beginning of the buffer so that Block Header + // along with Compressed Size and Uncompressed Size can be + // written there. + lzma_ret ret = lzma_block_header_size(&thr->block_options); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + // Initialize the Block encoder. + ret = lzma_block_encoder_init(&thr->block_encoder, + thr->allocator, &thr->block_options); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + size_t in_pos = 0; + size_t in_size = 0; + + thr->outbuf->size = thr->block_options.header_size; + const size_t out_size = thr->coder->outq.buf_size_max; + + do { + mythread_sync(thr->mutex) { + // Store in_pos and out_pos into *thr so that + // an application may read them via + // lzma_get_progress() to get progress information. + // + // NOTE: These aren't updated when the encoding + // finishes. Instead, the final values are taken + // later from thr->outbuf. + thr->progress_in = in_pos; + thr->progress_out = thr->outbuf->size; + + while (in_size == thr->in_size + && thr->state == THR_RUN) + mythread_cond_wait(&thr->cond, &thr->mutex); + + state = thr->state; + in_size = thr->in_size; + } + + // Return if we were asked to stop or exit. + if (state >= THR_STOP) + return state; + + lzma_action action = state == THR_FINISH + ? LZMA_FINISH : LZMA_RUN; + + // Limit the amount of input given to the Block encoder + // at once. This way this thread can react fairly quickly + // if the main thread wants us to stop or exit. + static const size_t in_chunk_max = 16384; + size_t in_limit = in_size; + if (in_size - in_pos > in_chunk_max) { + in_limit = in_pos + in_chunk_max; + action = LZMA_RUN; + } + + ret = thr->block_encoder.code( + thr->block_encoder.coder, thr->allocator, + thr->in, &in_pos, in_limit, thr->outbuf->buf, + &thr->outbuf->size, out_size, action); + } while (ret == LZMA_OK && thr->outbuf->size < out_size); + + switch (ret) { + case LZMA_STREAM_END: + assert(state == THR_FINISH); + + // Encode the Block Header. By doing it after + // the compression, we can store the Compressed Size + // and Uncompressed Size fields. + ret = lzma_block_header_encode(&thr->block_options, + thr->outbuf->buf); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + break; + + case LZMA_OK: + // The data was incompressible. Encode it using uncompressed + // LZMA2 chunks. + // + // First wait that we have gotten all the input. + mythread_sync(thr->mutex) { + while (thr->state == THR_RUN) + mythread_cond_wait(&thr->cond, &thr->mutex); + + state = thr->state; + in_size = thr->in_size; + } + + if (state >= THR_STOP) + return state; + + // Do the encoding. This takes care of the Block Header too. + thr->outbuf->size = 0; + ret = lzma_block_uncomp_encode(&thr->block_options, + thr->in, in_size, thr->outbuf->buf, + &thr->outbuf->size, out_size); + + // It shouldn't fail. + if (ret != LZMA_OK) { + worker_error(thr, LZMA_PROG_ERROR); + return THR_STOP; + } + + break; + + default: + worker_error(thr, ret); + return THR_STOP; + } + + // Set the size information that will be read by the main thread + // to write the Index field. + thr->outbuf->unpadded_size + = lzma_block_unpadded_size(&thr->block_options); + assert(thr->outbuf->unpadded_size != 0); + thr->outbuf->uncompressed_size = thr->block_options.uncompressed_size; + + return THR_FINISH; +} + + +static MYTHREAD_RET_TYPE +worker_start(void *thr_ptr) +{ + worker_thread *thr = thr_ptr; + worker_state state = THR_IDLE; // Init to silence a warning + + while (true) { + // Wait for work. + mythread_sync(thr->mutex) { + while (true) { + // The thread is already idle so if we are + // requested to stop, just set the state. + if (thr->state == THR_STOP) { + thr->state = THR_IDLE; + mythread_cond_signal(&thr->cond); + } + + state = thr->state; + if (state != THR_IDLE) + break; + + mythread_cond_wait(&thr->cond, &thr->mutex); + } + } + + assert(state != THR_IDLE); + assert(state != THR_STOP); + + if (state <= THR_FINISH) + state = worker_encode(thr, state); + + if (state == THR_EXIT) + break; + + // Mark the thread as idle unless the main thread has + // told us to exit. Signal is needed for the case + // where the main thread is waiting for the threads to stop. + mythread_sync(thr->mutex) { + if (thr->state != THR_EXIT) { + thr->state = THR_IDLE; + mythread_cond_signal(&thr->cond); + } + } + + mythread_sync(thr->coder->mutex) { + // Mark the output buffer as finished if + // no errors occurred. + thr->outbuf->finished = state == THR_FINISH; + + // Update the main progress info. + thr->coder->progress_in + += thr->outbuf->uncompressed_size; + thr->coder->progress_out += thr->outbuf->size; + thr->progress_in = 0; + thr->progress_out = 0; + + // Return this thread to the stack of free threads. + thr->next = thr->coder->threads_free; + thr->coder->threads_free = thr; + + mythread_cond_signal(&thr->coder->cond); + } + } + + // Exiting, free the resources. + mythread_mutex_destroy(&thr->mutex); + mythread_cond_destroy(&thr->cond); + + lzma_next_end(&thr->block_encoder, thr->allocator); + lzma_free(thr->in, thr->allocator); + return MYTHREAD_RET_VALUE; +} + + +/// Make the threads stop but not exit. Optionally wait for them to stop. +static void +threads_stop(lzma_stream_coder *coder, bool wait_for_threads) +{ + // Tell the threads to stop. + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + coder->threads[i].state = THR_STOP; + mythread_cond_signal(&coder->threads[i].cond); + } + } + + if (!wait_for_threads) + return; + + // Wait for the threads to settle in the idle state. + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + while (coder->threads[i].state != THR_IDLE) + mythread_cond_wait(&coder->threads[i].cond, + &coder->threads[i].mutex); + } + } + + return; +} + + +/// Stop the threads and free the resources associated with them. +/// Wait until the threads have exited. +static void +threads_end(lzma_stream_coder *coder, const lzma_allocator *allocator) +{ + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + coder->threads[i].state = THR_EXIT; + mythread_cond_signal(&coder->threads[i].cond); + } + } + + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + int ret = mythread_join(coder->threads[i].thread_id); + assert(ret == 0); + (void)ret; + } + + lzma_free(coder->threads, allocator); + return; +} + + +/// Initialize a new worker_thread structure and create a new thread. +static lzma_ret +initialize_new_thread(lzma_stream_coder *coder, + const lzma_allocator *allocator) +{ + worker_thread *thr = &coder->threads[coder->threads_initialized]; + + thr->in = lzma_alloc(coder->block_size, allocator); + if (thr->in == NULL) + return LZMA_MEM_ERROR; + + if (mythread_mutex_init(&thr->mutex)) + goto error_mutex; + + if (mythread_cond_init(&thr->cond)) + goto error_cond; + + thr->state = THR_IDLE; + thr->allocator = allocator; + thr->coder = coder; + thr->progress_in = 0; + thr->progress_out = 0; + thr->block_encoder = LZMA_NEXT_CODER_INIT; + + if (mythread_create(&thr->thread_id, &worker_start, thr)) + goto error_thread; + + ++coder->threads_initialized; + coder->thr = thr; + + return LZMA_OK; + +error_thread: + mythread_cond_destroy(&thr->cond); + +error_cond: + mythread_mutex_destroy(&thr->mutex); + +error_mutex: + lzma_free(thr->in, allocator); + return LZMA_MEM_ERROR; +} + + +static lzma_ret +get_thread(lzma_stream_coder *coder, const lzma_allocator *allocator) +{ + // If there are no free output subqueues, there is no + // point to try getting a thread. + if (!lzma_outq_has_buf(&coder->outq)) + return LZMA_OK; + + // If there is a free structure on the stack, use it. + mythread_sync(coder->mutex) { + if (coder->threads_free != NULL) { + coder->thr = coder->threads_free; + coder->threads_free = coder->threads_free->next; + } + } + + if (coder->thr == NULL) { + // If there are no uninitialized structures left, return. + if (coder->threads_initialized == coder->threads_max) + return LZMA_OK; + + // Initialize a new thread. + return_if_error(initialize_new_thread(coder, allocator)); + } + + // Reset the parts of the thread state that have to be done + // in the main thread. + mythread_sync(coder->thr->mutex) { + coder->thr->state = THR_RUN; + coder->thr->in_size = 0; + coder->thr->outbuf = lzma_outq_get_buf(&coder->outq); + mythread_cond_signal(&coder->thr->cond); + } + + return LZMA_OK; +} + + +static lzma_ret +stream_encode_in(lzma_stream_coder *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, lzma_action action) +{ + while (*in_pos < in_size + || (coder->thr != NULL && action != LZMA_RUN)) { + if (coder->thr == NULL) { + // Get a new thread. + const lzma_ret ret = get_thread(coder, allocator); + if (coder->thr == NULL) + return ret; + } + + // Copy the input data to thread's buffer. + size_t thr_in_size = coder->thr->in_size; + lzma_bufcpy(in, in_pos, in_size, coder->thr->in, + &thr_in_size, coder->block_size); + + // Tell the Block encoder to finish if + // - it has got block_size bytes of input; or + // - all input was used and LZMA_FINISH, LZMA_FULL_FLUSH, + // or LZMA_FULL_BARRIER was used. + // + // TODO: LZMA_SYNC_FLUSH and LZMA_SYNC_BARRIER. + const bool finish = thr_in_size == coder->block_size + || (*in_pos == in_size && action != LZMA_RUN); + + bool block_error = false; + + mythread_sync(coder->thr->mutex) { + if (coder->thr->state == THR_IDLE) { + // Something has gone wrong with the Block + // encoder. It has set coder->thread_error + // which we will read a few lines later. + block_error = true; + } else { + // Tell the Block encoder its new amount + // of input and update the state if needed. + coder->thr->in_size = thr_in_size; + + if (finish) + coder->thr->state = THR_FINISH; + + mythread_cond_signal(&coder->thr->cond); + } + } + + if (block_error) { + lzma_ret ret; + + mythread_sync(coder->mutex) { + ret = coder->thread_error; + } + + return ret; + } + + if (finish) + coder->thr = NULL; + } + + return LZMA_OK; +} + + +/// Wait until more input can be consumed, more output can be read, or +/// an optional timeout is reached. +static bool +wait_for_work(lzma_stream_coder *coder, mythread_condtime *wait_abs, + bool *has_blocked, bool has_input) +{ + if (coder->timeout != 0 && !*has_blocked) { + // Every time when stream_encode_mt() is called via + // lzma_code(), *has_blocked starts as false. We set it + // to true here and calculate the absolute time when + // we must return if there's nothing to do. + // + // The idea of *has_blocked is to avoid unneeded calls + // to mythread_condtime_set(), which may do a syscall + // depending on the operating system. + *has_blocked = true; + mythread_condtime_set(wait_abs, &coder->cond, coder->timeout); + } + + bool timed_out = false; + + mythread_sync(coder->mutex) { + // There are four things that we wait. If one of them + // becomes possible, we return. + // - If there is input left, we need to get a free + // worker thread and an output buffer for it. + // - Data ready to be read from the output queue. + // - A worker thread indicates an error. + // - Time out occurs. + while ((!has_input || coder->threads_free == NULL + || !lzma_outq_has_buf(&coder->outq)) + && !lzma_outq_is_readable(&coder->outq) + && coder->thread_error == LZMA_OK + && !timed_out) { + if (coder->timeout != 0) + timed_out = mythread_cond_timedwait( + &coder->cond, &coder->mutex, + wait_abs) != 0; + else + mythread_cond_wait(&coder->cond, + &coder->mutex); + } + } + + return timed_out; +} + + +static lzma_ret +stream_encode_mt(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + lzma_stream_coder *coder = coder_ptr; + + switch (coder->sequence) { + case SEQ_STREAM_HEADER: + lzma_bufcpy(coder->header, &coder->header_pos, + sizeof(coder->header), + out, out_pos, out_size); + if (coder->header_pos < sizeof(coder->header)) + return LZMA_OK; + + coder->header_pos = 0; + coder->sequence = SEQ_BLOCK; + + // Fall through + + case SEQ_BLOCK: { + // Initialized to silence warnings. + lzma_vli unpadded_size = 0; + lzma_vli uncompressed_size = 0; + lzma_ret ret = LZMA_OK; + + // These are for wait_for_work(). + bool has_blocked = false; + mythread_condtime wait_abs; + + while (true) { + mythread_sync(coder->mutex) { + // Check for Block encoder errors. + ret = coder->thread_error; + if (ret != LZMA_OK) { + assert(ret != LZMA_STREAM_END); + break; + } + + // Try to read compressed data to out[]. + ret = lzma_outq_read(&coder->outq, + out, out_pos, out_size, + &unpadded_size, + &uncompressed_size); + } + + if (ret == LZMA_STREAM_END) { + // End of Block. Add it to the Index. + ret = lzma_index_append(coder->index, + allocator, unpadded_size, + uncompressed_size); + + // If we didn't fill the output buffer yet, + // try to read more data. Maybe the next + // outbuf has been finished already too. + if (*out_pos < out_size) + continue; + } + + if (ret != LZMA_OK) { + // coder->thread_error was set or + // lzma_index_append() failed. + threads_stop(coder, false); + return ret; + } + + // Try to give uncompressed data to a worker thread. + ret = stream_encode_in(coder, allocator, + in, in_pos, in_size, action); + if (ret != LZMA_OK) { + threads_stop(coder, false); + return ret; + } + + // See if we should wait or return. + // + // TODO: LZMA_SYNC_FLUSH and LZMA_SYNC_BARRIER. + if (*in_pos == in_size) { + // LZMA_RUN: More data is probably coming + // so return to let the caller fill the + // input buffer. + if (action == LZMA_RUN) + return LZMA_OK; + + // LZMA_FULL_BARRIER: The same as with + // LZMA_RUN but tell the caller that the + // barrier was completed. + if (action == LZMA_FULL_BARRIER) + return LZMA_STREAM_END; + + // Finishing or flushing isn't completed until + // all input data has been encoded and copied + // to the output buffer. + if (lzma_outq_is_empty(&coder->outq)) { + // LZMA_FINISH: Continue to encode + // the Index field. + if (action == LZMA_FINISH) + break; + + // LZMA_FULL_FLUSH: Return to tell + // the caller that flushing was + // completed. + if (action == LZMA_FULL_FLUSH) + return LZMA_STREAM_END; + } + } + + // Return if there is no output space left. + // This check must be done after testing the input + // buffer, because we might want to use a different + // return code. + if (*out_pos == out_size) + return LZMA_OK; + + // Neither in nor out has been used completely. + // Wait until there's something we can do. + if (wait_for_work(coder, &wait_abs, &has_blocked, + *in_pos < in_size)) + return LZMA_TIMED_OUT; + } + + // All Blocks have been encoded and the threads have stopped. + // Prepare to encode the Index field. + return_if_error(lzma_index_encoder_init( + &coder->index_encoder, allocator, + coder->index)); + coder->sequence = SEQ_INDEX; + + // Update the progress info to take the Index and + // Stream Footer into account. Those are very fast to encode + // so in terms of progress information they can be thought + // to be ready to be copied out. + coder->progress_out += lzma_index_size(coder->index) + + LZMA_STREAM_HEADER_SIZE; + } + + // Fall through + + case SEQ_INDEX: { + // Call the Index encoder. It doesn't take any input, so + // those pointers can be NULL. + const lzma_ret ret = coder->index_encoder.code( + coder->index_encoder.coder, allocator, + NULL, NULL, 0, + out, out_pos, out_size, LZMA_RUN); + if (ret != LZMA_STREAM_END) + return ret; + + // Encode the Stream Footer into coder->buffer. + coder->stream_flags.backward_size + = lzma_index_size(coder->index); + if (lzma_stream_footer_encode(&coder->stream_flags, + coder->header) != LZMA_OK) + return LZMA_PROG_ERROR; + + coder->sequence = SEQ_STREAM_FOOTER; + } + + // Fall through + + case SEQ_STREAM_FOOTER: + lzma_bufcpy(coder->header, &coder->header_pos, + sizeof(coder->header), + out, out_pos, out_size); + return coder->header_pos < sizeof(coder->header) + ? LZMA_OK : LZMA_STREAM_END; + } + + assert(0); + return LZMA_PROG_ERROR; +} + + +static void +stream_encoder_mt_end(void *coder_ptr, const lzma_allocator *allocator) +{ + lzma_stream_coder *coder = coder_ptr; + + // Threads must be killed before the output queue can be freed. + threads_end(coder, allocator); + lzma_outq_end(&coder->outq, allocator); + + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + lzma_free(coder->filters[i].options, allocator); + + lzma_next_end(&coder->index_encoder, allocator); + lzma_index_end(coder->index, allocator); + + mythread_cond_destroy(&coder->cond); + mythread_mutex_destroy(&coder->mutex); + + lzma_free(coder, allocator); + return; +} + + +/// Options handling for lzma_stream_encoder_mt_init() and +/// lzma_stream_encoder_mt_memusage() +static lzma_ret +get_options(const lzma_mt *options, lzma_options_easy *opt_easy, + const lzma_filter **filters, uint64_t *block_size, + uint64_t *outbuf_size_max) +{ + // Validate some of the options. + if (options == NULL) + return LZMA_PROG_ERROR; + + if (options->flags != 0 || options->threads == 0 + || options->threads > LZMA_THREADS_MAX) + return LZMA_OPTIONS_ERROR; + + if (options->filters != NULL) { + // Filter chain was given, use it as is. + *filters = options->filters; + } else { + // Use a preset. + if (lzma_easy_preset(opt_easy, options->preset)) + return LZMA_OPTIONS_ERROR; + + *filters = opt_easy->filters; + } + + // Block size + if (options->block_size > 0) { + if (options->block_size > BLOCK_SIZE_MAX) + return LZMA_OPTIONS_ERROR; + + *block_size = options->block_size; + } else { + // Determine the Block size from the filter chain. + *block_size = lzma_mt_block_size(*filters); + if (*block_size == 0) + return LZMA_OPTIONS_ERROR; + + assert(*block_size <= BLOCK_SIZE_MAX); + } + + // Calculate the maximum amount output that a single output buffer + // may need to hold. This is the same as the maximum total size of + // a Block. + *outbuf_size_max = lzma_block_buffer_bound64(*block_size); + if (*outbuf_size_max == 0) + return LZMA_MEM_ERROR; + + return LZMA_OK; +} + + +static void +get_progress(void *coder_ptr, uint64_t *progress_in, uint64_t *progress_out) +{ + lzma_stream_coder *coder = coder_ptr; + + // Lock coder->mutex to prevent finishing threads from moving their + // progress info from the worker_thread structure to lzma_stream_coder. + mythread_sync(coder->mutex) { + *progress_in = coder->progress_in; + *progress_out = coder->progress_out; + + for (size_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + *progress_in += coder->threads[i].progress_in; + *progress_out += coder->threads[i] + .progress_out; + } + } + } + + return; +} + + +static lzma_ret +stream_encoder_mt_init(lzma_next_coder *next, const lzma_allocator *allocator, + const lzma_mt *options) +{ + lzma_next_coder_init(&stream_encoder_mt_init, next, allocator); + + // Get the filter chain. + lzma_options_easy easy; + const lzma_filter *filters; + uint64_t block_size; + uint64_t outbuf_size_max; + return_if_error(get_options(options, &easy, &filters, + &block_size, &outbuf_size_max)); + +#if SIZE_MAX < UINT64_MAX + if (block_size > SIZE_MAX) + return LZMA_MEM_ERROR; +#endif + + // Validate the filter chain so that we can give an error in this + // function instead of delaying it to the first call to lzma_code(). + // The memory usage calculation verifies the filter chain as + // a side effect so we take advatange of that. + if (lzma_raw_encoder_memusage(filters) == UINT64_MAX) + return LZMA_OPTIONS_ERROR; + + // Validate the Check ID. + if ((unsigned int)(options->check) > LZMA_CHECK_ID_MAX) + return LZMA_PROG_ERROR; + + if (!lzma_check_is_supported(options->check)) + return LZMA_UNSUPPORTED_CHECK; + + // Allocate and initialize the base structure if needed. + lzma_stream_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) + return LZMA_MEM_ERROR; + + next->coder = coder; + + // For the mutex and condition variable initializations + // the error handling has to be done here because + // stream_encoder_mt_end() doesn't know if they have + // already been initialized or not. + if (mythread_mutex_init(&coder->mutex)) { + lzma_free(coder, allocator); + next->coder = NULL; + return LZMA_MEM_ERROR; + } + + if (mythread_cond_init(&coder->cond)) { + mythread_mutex_destroy(&coder->mutex); + lzma_free(coder, allocator); + next->coder = NULL; + return LZMA_MEM_ERROR; + } + + next->code = &stream_encode_mt; + next->end = &stream_encoder_mt_end; + next->get_progress = &get_progress; +// next->update = &stream_encoder_mt_update; + + coder->filters[0].id = LZMA_VLI_UNKNOWN; + coder->index_encoder = LZMA_NEXT_CODER_INIT; + coder->index = NULL; + memzero(&coder->outq, sizeof(coder->outq)); + coder->threads = NULL; + coder->threads_max = 0; + coder->threads_initialized = 0; + } + + // Basic initializations + coder->sequence = SEQ_STREAM_HEADER; + coder->block_size = (size_t)(block_size); + coder->thread_error = LZMA_OK; + coder->thr = NULL; + + // Allocate the thread-specific base structures. + assert(options->threads > 0); + if (coder->threads_max != options->threads) { + threads_end(coder, allocator); + + coder->threads = NULL; + coder->threads_max = 0; + + coder->threads_initialized = 0; + coder->threads_free = NULL; + + coder->threads = lzma_alloc( + options->threads * sizeof(worker_thread), + allocator); + if (coder->threads == NULL) + return LZMA_MEM_ERROR; + + coder->threads_max = options->threads; + } else { + // Reuse the old structures and threads. Tell the running + // threads to stop and wait until they have stopped. + threads_stop(coder, true); + } + + // Output queue + return_if_error(lzma_outq_init(&coder->outq, allocator, + outbuf_size_max, options->threads)); + + // Timeout + coder->timeout = options->timeout; + + // Free the old filter chain and copy the new one. + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + lzma_free(coder->filters[i].options, allocator); + + return_if_error(lzma_filters_copy( + filters, coder->filters, allocator)); + + // Index + lzma_index_end(coder->index, allocator); + coder->index = lzma_index_init(allocator); + if (coder->index == NULL) + return LZMA_MEM_ERROR; + + // Stream Header + coder->stream_flags.version = 0; + coder->stream_flags.check = options->check; + return_if_error(lzma_stream_header_encode( + &coder->stream_flags, coder->header)); + + coder->header_pos = 0; + + // Progress info + coder->progress_in = 0; + coder->progress_out = LZMA_STREAM_HEADER_SIZE; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_stream_encoder_mt(lzma_stream *strm, const lzma_mt *options) +{ + lzma_next_strm_init(stream_encoder_mt_init, strm, options); + + strm->internal->supported_actions[LZMA_RUN] = true; +// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_BARRIER] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} + + +// This function name is a monster but it's consistent with the older +// monster names. :-( 31 chars is the max that C99 requires so in that +// sense it's not too long. ;-) +extern LZMA_API(uint64_t) +lzma_stream_encoder_mt_memusage(const lzma_mt *options) +{ + lzma_options_easy easy; + const lzma_filter *filters; + uint64_t block_size; + uint64_t outbuf_size_max; + + if (get_options(options, &easy, &filters, &block_size, + &outbuf_size_max) != LZMA_OK) + return UINT64_MAX; + + // Memory usage of the input buffers + const uint64_t inbuf_memusage = options->threads * block_size; + + // Memory usage of the filter encoders + uint64_t filters_memusage = lzma_raw_encoder_memusage(filters); + if (filters_memusage == UINT64_MAX) + return UINT64_MAX; + + filters_memusage *= options->threads; + + // Memory usage of the output queue + const uint64_t outq_memusage = lzma_outq_memusage( + outbuf_size_max, options->threads); + if (outq_memusage == UINT64_MAX) + return UINT64_MAX; + + // Sum them with overflow checking. + uint64_t total_memusage = LZMA_MEMUSAGE_BASE + + sizeof(lzma_stream_coder) + + options->threads * sizeof(worker_thread); + + if (UINT64_MAX - total_memusage < inbuf_memusage) + return UINT64_MAX; + + total_memusage += inbuf_memusage; + + if (UINT64_MAX - total_memusage < filters_memusage) + return UINT64_MAX; + + total_memusage += filters_memusage; + + if (UINT64_MAX - total_memusage < outq_memusage) + return UINT64_MAX; + + return total_memusage + outq_memusage; +} diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c index 8cf48a4..1bc2f97 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c @@ -30,15 +30,13 @@ stream_flags_decode(lzma_stream_flags *options, const uint8_t *in) extern LZMA_API(lzma_ret) lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in) { - uint32_t crc; - // Magic if (memcmp(in, lzma_header_magic, sizeof(lzma_header_magic)) != 0) return LZMA_FORMAT_ERROR; // Verify the CRC32 so we can distinguish between corrupt // and unsupported files. - crc = lzma_crc32(in + sizeof(lzma_header_magic), + const uint32_t crc = lzma_crc32(in + sizeof(lzma_header_magic), LZMA_STREAM_FLAGS_SIZE, 0); if (crc != unaligned_read32le(in + sizeof(lzma_header_magic) + LZMA_STREAM_FLAGS_SIZE)) @@ -61,15 +59,13 @@ lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in) extern LZMA_API(lzma_ret) lzma_stream_footer_decode(lzma_stream_flags *options, const uint8_t *in) { - uint32_t crc; - // Magic if (memcmp(in + sizeof(uint32_t) * 2 + LZMA_STREAM_FLAGS_SIZE, lzma_footer_magic, sizeof(lzma_footer_magic)) != 0) return LZMA_FORMAT_ERROR; // CRC32 - crc = lzma_crc32(in + sizeof(uint32_t), + const uint32_t crc = lzma_crc32(in + sizeof(uint32_t), sizeof(uint32_t) + LZMA_STREAM_FLAGS_SIZE, 0); if (crc != unaligned_read32le(in)) return LZMA_DATA_ERROR; diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c index 290339e..4e71715 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c @@ -29,8 +29,6 @@ stream_flags_encode(const lzma_stream_flags *options, uint8_t *out) extern LZMA_API(lzma_ret) lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) { - uint32_t crc; - assert(sizeof(lzma_header_magic) + LZMA_STREAM_FLAGS_SIZE + 4 == LZMA_STREAM_HEADER_SIZE); @@ -45,7 +43,7 @@ lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) return LZMA_PROG_ERROR; // CRC32 of the Stream Header - crc = lzma_crc32(out + sizeof(lzma_header_magic), + const uint32_t crc = lzma_crc32(out + sizeof(lzma_header_magic), LZMA_STREAM_FLAGS_SIZE, 0); unaligned_write32le(out + sizeof(lzma_header_magic) @@ -58,8 +56,6 @@ lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) extern LZMA_API(lzma_ret) lzma_stream_footer_encode(const lzma_stream_flags *options, uint8_t *out) { - uint32_t crc; - assert(2 * 4 + LZMA_STREAM_FLAGS_SIZE + sizeof(lzma_footer_magic) == LZMA_STREAM_HEADER_SIZE); @@ -77,7 +73,7 @@ lzma_stream_footer_encode(const lzma_stream_flags *options, uint8_t *out) return LZMA_PROG_ERROR; // CRC32 - crc = lzma_crc32( + const uint32_t crc = lzma_crc32( out + 4, 4 + LZMA_STREAM_FLAGS_SIZE, 0); unaligned_write32le(out, crc); diff --git a/Utilities/cmliblzma/liblzma/common/vli_decoder.c b/Utilities/cmliblzma/liblzma/common/vli_decoder.c index 1c66384..c181828 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/vli_decoder.c @@ -14,8 +14,8 @@ extern LZMA_API(lzma_ret) -lzma_vli_decode(lzma_vli *LZMA_RESTRICT vli, size_t *vli_pos, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lzma_vli_decode(lzma_vli *restrict vli, size_t *vli_pos, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size) { // If we haven't been given vli_pos, work in single-call mode. diff --git a/Utilities/cmliblzma/liblzma/common/vli_encoder.c b/Utilities/cmliblzma/liblzma/common/vli_encoder.c index 09e90cb..f864269 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/vli_encoder.c @@ -15,7 +15,7 @@ extern LZMA_API(lzma_ret) lzma_vli_encode(lzma_vli vli, size_t *vli_pos, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { // If we haven't been given vli_pos, work in single-call mode. diff --git a/Utilities/cmliblzma/liblzma/common/vli_size.c b/Utilities/cmliblzma/liblzma/common/vli_size.c index 8b931e4..ec1b4fa 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_size.c +++ b/Utilities/cmliblzma/liblzma/common/vli_size.c @@ -16,11 +16,10 @@ extern LZMA_API(uint32_t) lzma_vli_size(lzma_vli vli) { - uint32_t i = 0; - if (vli > LZMA_VLI_MAX) return 0; + uint32_t i = 0; do { vli >>= 7; ++i; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_common.c b/Utilities/cmliblzma/liblzma/delta/delta_common.c index 803e674..4768201 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_common.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_common.c @@ -15,8 +15,9 @@ static void -delta_coder_end(lzma_coder *coder, lzma_allocator *allocator) +delta_coder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_delta_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -24,20 +25,21 @@ delta_coder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_delta_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { - const lzma_options_delta *opt; - // Allocate memory for the decoder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_delta_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_delta_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; + // End function is the same for encoder and decoder. next->end = &delta_coder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Validate the options. @@ -45,16 +47,15 @@ lzma_delta_coder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_OPTIONS_ERROR; // Set the delta distance. - opt = filters[0].options; - next->coder->distance = opt->dist; + const lzma_options_delta *opt = filters[0].options; + coder->distance = opt->dist; // Initialize the rest of the variables. - next->coder->pos = 0; - memzero(next->coder->history, LZMA_DELTA_DIST_MAX); + coder->pos = 0; + memzero(coder->history, LZMA_DELTA_DIST_MAX); // Initialize the next decoder in the chain, if any. - return lzma_next_filter_init(&next->coder->next, - allocator, filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } @@ -68,5 +69,5 @@ lzma_delta_coder_memusage(const void *options) || opt->dist > LZMA_DELTA_DIST_MAX) return UINT64_MAX; - return sizeof(lzma_coder); + return sizeof(lzma_delta_coder); } diff --git a/Utilities/cmliblzma/liblzma/delta/delta_decoder.c b/Utilities/cmliblzma/liblzma/delta/delta_decoder.c index 28df727..6859afa 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_decoder.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_decoder.c @@ -15,12 +15,11 @@ static void -decode_buffer(lzma_coder *coder, uint8_t *buffer, size_t size) +decode_buffer(lzma_delta_coder *coder, uint8_t *buffer, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { buffer[i] += coder->history[(distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = buffer[i]; } @@ -28,17 +27,18 @@ decode_buffer(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -delta_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +delta_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { - const size_t out_start = *out_pos; - lzma_ret ret; + lzma_delta_coder *coder = coder_ptr; assert(coder->next.code != NULL); - ret = coder->next.code(coder->next.coder, allocator, + const size_t out_start = *out_pos; + + const lzma_ret ret = coder->next.code(coder->next.coder, allocator, in, in_pos, in_size, out, out_pos, out_size, action); @@ -49,7 +49,7 @@ delta_decode(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { next->code = &delta_decode; @@ -58,15 +58,14 @@ lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_delta_props_decode(void **options, lzma_allocator *allocator, +lzma_delta_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_delta *opt; - if (props_size != 1) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_delta), allocator); + lzma_options_delta *opt + = lzma_alloc(sizeof(lzma_options_delta), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_decoder.h b/Utilities/cmliblzma/liblzma/delta/delta_decoder.h index ae89acc..ad89cc6 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_decoder.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_decoder.h @@ -16,10 +16,11 @@ #include "delta_common.h" extern lzma_ret lzma_delta_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_delta_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/delta/delta_encoder.c b/Utilities/cmliblzma/liblzma/delta/delta_encoder.c index a39c154..3841651 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_encoder.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_encoder.c @@ -18,13 +18,12 @@ /// is the first filter in the chain (and thus the last filter in the /// encoder's filter stack). static void -copy_and_encode(lzma_coder *coder, - const uint8_t *LZMA_RESTRICT in, uint8_t *LZMA_RESTRICT out, size_t size) +copy_and_encode(lzma_delta_coder *coder, + const uint8_t *restrict in, uint8_t *restrict out, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { const uint8_t tmp = coder->history[ (distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = in[i]; @@ -36,12 +35,11 @@ copy_and_encode(lzma_coder *coder, /// Encodes the data in place. This is used when we are the last filter /// in the chain (and thus non-last filter in the encoder's filter stack). static void -encode_in_place(lzma_coder *coder, uint8_t *buffer, size_t size) +encode_in_place(lzma_delta_coder *coder, uint8_t *buffer, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { const uint8_t tmp = coder->history[ (distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = buffer[i]; @@ -51,11 +49,13 @@ encode_in_place(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -delta_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +delta_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_delta_coder *coder = coder_ptr; + lzma_ret ret; if (coder->next.code == NULL) { @@ -86,10 +86,12 @@ delta_encode(lzma_coder *coder, lzma_allocator *allocator, static lzma_ret -delta_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +delta_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_delta_coder *coder = coder_ptr; + // Delta doesn't and will never support changing the options in // the middle of encoding. If the app tries to change them, we // simply ignore them. @@ -99,7 +101,7 @@ delta_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { next->code = &delta_encode; @@ -111,13 +113,12 @@ lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out) { - const lzma_options_delta *opt = options; - // The caller must have already validated the options, so it's // LZMA_PROG_ERROR if they are invalid. if (lzma_delta_coder_memusage(options) == UINT64_MAX) return LZMA_PROG_ERROR; + const lzma_options_delta *opt = options; out[0] = opt->dist - LZMA_DELTA_DIST_MIN; return LZMA_OK; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_encoder.h b/Utilities/cmliblzma/liblzma/delta/delta_encoder.h index a447862..4ab9847 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_encoder.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_encoder.h @@ -16,7 +16,8 @@ #include "delta_common.h" extern lzma_ret lzma_delta_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out); diff --git a/Utilities/cmliblzma/liblzma/delta/delta_private.h b/Utilities/cmliblzma/liblzma/delta/delta_private.h index 62b7fed..0d6cb38 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_private.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_private.h @@ -15,7 +15,7 @@ #include "delta_common.h" -struct lzma_coder_s { +typedef struct { /// Next coder in the chain lzma_next_coder next; @@ -27,11 +27,11 @@ struct lzma_coder_s { /// Buffer to hold history of the original data uint8_t history[LZMA_DELTA_DIST_MAX]; -}; +} lzma_delta_coder; extern lzma_ret lzma_delta_coder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/liblzma.pc.in b/Utilities/cmliblzma/liblzma/liblzma.pc.in index 7f11f1a..9fa4891 100644 --- a/Utilities/cmliblzma/liblzma/liblzma.pc.in +++ b/Utilities/cmliblzma/liblzma/liblzma.pc.in @@ -16,4 +16,4 @@ URL: @PACKAGE_URL@ Version: @PACKAGE_VERSION@ Cflags: -I${includedir} Libs: -L${libdir} -llzma -Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ +Libs.private: @PTHREAD_CFLAGS@ @LIBS@ diff --git a/Utilities/cmliblzma/liblzma/lz/lz_decoder.c b/Utilities/cmliblzma/liblzma/lz/lz_decoder.c index 9fa1bdc..c708644 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_decoder.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_decoder.c @@ -20,7 +20,7 @@ #include "lz_decoder.h" -struct lzma_coder_s { +typedef struct { /// Dictionary (history buffer) lzma_dict dict; @@ -48,7 +48,7 @@ struct lzma_coder_s { size_t size; uint8_t buffer[LZMA_BUFFER_SIZE]; } temp; -}; +} lzma_coder; static void @@ -64,22 +64,18 @@ lz_decoder_reset(lzma_coder *coder) static lzma_ret decode_buffer(lzma_coder *coder, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size) + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) { while (true) { - size_t copy_size; - size_t dict_start; - lzma_ret ret; - // Wrap the dictionary if needed. if (coder->dict.pos == coder->dict.size) coder->dict.pos = 0; // Store the current dictionary position. It is needed to know // where to start copying to the out[] buffer. - dict_start = coder->dict.pos; + const size_t dict_start = coder->dict.pos; // Calculate how much we allow coder->lz.code() to decode. // It must not decode past the end of the dictionary @@ -90,13 +86,13 @@ decode_buffer(lzma_coder *coder, coder->dict.size - coder->dict.pos); // Call the coder->lz.code() to do the actual decoding. - ret = coder->lz.code( + const lzma_ret ret = coder->lz.code( coder->lz.coder, &coder->dict, in, in_pos, in_size); // Copy the decoded data from the dictionary to the out[] // buffer. - copy_size = coder->dict.pos - dict_start; + const size_t copy_size = coder->dict.pos - dict_start; assert(copy_size <= out_size - *out_pos); memcpy(out + *out_pos, coder->dict.buf + dict_start, copy_size); @@ -129,13 +125,15 @@ decode_buffer(lzma_coder *coder, static lzma_ret -lz_decode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +lz_decode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_coder *coder = coder_ptr; + if (coder->next.code == NULL) return decode_buffer(coder, in, in_pos, in_size, out, out_pos, out_size); @@ -143,15 +141,13 @@ lz_decode(lzma_coder *coder, // We aren't the last coder in the chain, we need to decode // our input to a temporary buffer. while (*out_pos < out_size) { - lzma_ret ret; - // Fill the temporary buffer if it is empty. if (!coder->next_finished && coder->temp.pos == coder->temp.size) { coder->temp.pos = 0; coder->temp.size = 0; - ret = coder->next.code( + const lzma_ret ret = coder->next.code( coder->next.coder, allocator, in, in_pos, in_size, coder->temp.buffer, &coder->temp.size, @@ -173,7 +169,7 @@ lz_decode(lzma_coder *coder, return LZMA_OK; } - ret = decode_buffer(coder, coder->temp.buffer, + const lzma_ret ret = decode_buffer(coder, coder->temp.buffer, &coder->temp.pos, coder->temp.size, out, out_pos, out_size); @@ -190,8 +186,10 @@ lz_decode(lzma_coder *coder, static void -lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +lz_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_coder *coder = coder_ptr; + lzma_next_end(&coder->next, allocator); lzma_free(coder->dict.buf, allocator); @@ -206,32 +204,33 @@ lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_decoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)) { - lzma_lz_options lz_options; - // Allocate the base structure if it isn't already allocated. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &lz_decode; next->end = &lz_decoder_end; - next->coder->dict.buf = NULL; - next->coder->dict.size = 0; - next->coder->lz = LZMA_LZ_DECODER_INIT; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->dict.buf = NULL; + coder->dict.size = 0; + coder->lz = LZMA_LZ_DECODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Allocate and initialize the LZ-based decoder. It will also give // us the dictionary size. - return_if_error(lz_init(&next->coder->lz, allocator, + lzma_lz_options lz_options; + return_if_error(lz_init(&coder->lz, allocator, filters[0].options, &lz_options)); // If the dictionary size is very small, increase it to 4096 bytes. @@ -255,14 +254,14 @@ lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, lz_options.dict_size = (lz_options.dict_size + 15) & ~((size_t)(15)); // Allocate and initialize the dictionary. - if (next->coder->dict.size != lz_options.dict_size) { - lzma_free(next->coder->dict.buf, allocator); - next->coder->dict.buf + if (coder->dict.size != lz_options.dict_size) { + lzma_free(coder->dict.buf, allocator); + coder->dict.buf = lzma_alloc(lz_options.dict_size, allocator); - if (next->coder->dict.buf == NULL) + if (coder->dict.buf == NULL) return LZMA_MEM_ERROR; - next->coder->dict.size = lz_options.dict_size; + coder->dict.size = lz_options.dict_size; } lz_decoder_reset(next->coder); @@ -275,21 +274,20 @@ lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, const size_t copy_size = my_min(lz_options.preset_dict_size, lz_options.dict_size); const size_t offset = lz_options.preset_dict_size - copy_size; - memcpy(next->coder->dict.buf, lz_options.preset_dict + offset, + memcpy(coder->dict.buf, lz_options.preset_dict + offset, copy_size); - next->coder->dict.pos = copy_size; - next->coder->dict.full = copy_size; + coder->dict.pos = copy_size; + coder->dict.full = copy_size; } // Miscellaneous initializations - next->coder->next_finished = false; - next->coder->this_finished = false; - next->coder->temp.pos = 0; - next->coder->temp.size = 0; + coder->next_finished = false; + coder->this_finished = false; + coder->temp.pos = 0; + coder->temp.size = 0; // Initialize the next filter in the chain, if any. - return lzma_next_filter_init(&next->coder->next, allocator, - filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } @@ -301,7 +299,8 @@ lzma_lz_decoder_memusage(size_t dictionary_size) extern void -lzma_lz_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) +lzma_lz_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) { + lzma_coder *coder = coder_ptr; coder->lz.set_uncompressed(coder->lz.coder, uncompressed_size); } diff --git a/Utilities/cmliblzma/liblzma/lz/lz_decoder.h b/Utilities/cmliblzma/liblzma/lz/lz_decoder.h index 76011f2..754ccf3 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_decoder.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_decoder.h @@ -53,45 +53,45 @@ typedef struct { typedef struct { /// Data specific to the LZ-based decoder - lzma_coder *coder; + void *coder; /// Function to decode from in[] to *dict - lzma_ret (*code)(lzma_coder *LZMA_RESTRICT coder, - lzma_dict *LZMA_RESTRICT dict, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size); + lzma_ret (*code)(void *coder, + lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size); - void (*reset)(lzma_coder *coder, const void *options); + void (*reset)(void *coder, const void *options); /// Set the uncompressed size - void (*set_uncompressed)(lzma_coder *coder, - lzma_vli uncompressed_size); + void (*set_uncompressed)(void *coder, lzma_vli uncompressed_size); /// Free allocated resources - void (*end)(lzma_coder *coder, lzma_allocator *allocator); + void (*end)(void *coder, const lzma_allocator *allocator); } lzma_lz_decoder; -static const lzma_lz_decoder LZMA_LZ_DECODER_INIT = - { - NULL, - NULL, - NULL, - NULL, - NULL, - }; +#define LZMA_LZ_DECODER_INIT \ + (lzma_lz_decoder){ \ + .coder = NULL, \ + .code = NULL, \ + .reset = NULL, \ + .set_uncompressed = NULL, \ + .end = NULL, \ + } extern lzma_ret lzma_lz_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters, + const lzma_allocator *allocator, + const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_decoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)); extern uint64_t lzma_lz_decoder_memusage(size_t dictionary_size); extern void lzma_lz_decoder_uncompressed( - lzma_coder *coder, lzma_vli uncompressed_size); + void *coder, lzma_vli uncompressed_size); ////////////////////// @@ -151,15 +151,13 @@ dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len) dict->pos += left; } else { - uint32_t copy_pos; - uint32_t copy_size; - // The bigger the dictionary, the more rare this // case occurs. We need to "wrap" the dict, thus // we might need two memcpy() to copy all the data. assert(dict->full == dict->size); - copy_pos = dict->pos - distance - 1 + dict->size; - copy_size = dict->size - copy_pos; + const uint32_t copy_pos + = dict->pos - distance - 1 + dict->size; + uint32_t copy_size = dict->size - copy_pos; if (copy_size < left) { memmove(dict->buf + dict->pos, dict->buf + copy_pos, @@ -202,9 +200,9 @@ dict_put(lzma_dict *dict, uint8_t byte) /// Copies arbitrary amount of data into the dictionary. static inline void -dict_write(lzma_dict *LZMA_RESTRICT dict, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size, - size_t *LZMA_RESTRICT left) +dict_write(lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size, + size_t *restrict left) { // NOTE: If we are being given more data than the size of the // dictionary, it could be possible to optimize the LZ decoder diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder.c b/Utilities/cmliblzma/liblzma/lz/lz_encoder.c index 1dae924..9a74b7c 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder.c @@ -20,8 +20,10 @@ # include "lz_encoder_hash_table.h" #endif +#include "memcmplen.h" -struct lzma_coder_s { + +typedef struct { /// LZ-based encoder e.g. LZMA lzma_lz_encoder lz; @@ -30,7 +32,7 @@ struct lzma_coder_s { /// Next coder in the chain lzma_next_coder next; -}; +} lzma_coder; /// \brief Moves the data in the input window to free space for new data @@ -43,18 +45,16 @@ struct lzma_coder_s { static void move_window(lzma_mf *mf) { - uint32_t move_offset; - size_t move_size; - // Align the move to a multiple of 16 bytes. Some LZ-based encoders // like LZMA use the lowest bits of mf->read_pos to know the // alignment of the uncompressed data. We also get better speed // for memmove() with aligned buffers. assert(mf->read_pos > mf->keep_size_before); - move_offset = (mf->read_pos - mf->keep_size_before) & ~UINT32_C(15); + const uint32_t move_offset + = (mf->read_pos - mf->keep_size_before) & ~UINT32_C(15); assert(mf->write_pos > move_offset); - move_size = mf->write_pos - move_offset; + const size_t move_size = mf->write_pos - move_offset; assert(move_offset + move_size <= mf->size); @@ -78,12 +78,10 @@ move_window(lzma_mf *mf) /// This function must not be called once it has returned LZMA_STREAM_END. /// static lzma_ret -fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, - size_t *in_pos, size_t in_size, lzma_action action) +fill_window(lzma_coder *coder, const lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size, + lzma_action action) { - size_t write_pos; - lzma_ret ret; - assert(coder->mf.read_pos <= coder->mf.write_pos); // Move the sliding window if needed. @@ -93,7 +91,8 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, // Maybe this is ugly, but lzma_mf uses uint32_t for most things // (which I find cleanest), but we need size_t here when filling // the history window. - write_pos = coder->mf.write_pos; + size_t write_pos = coder->mf.write_pos; + lzma_ret ret; if (coder->next.code == NULL) { // Not using a filter, simply memcpy() as much as possible. lzma_bufcpy(in, in_pos, in_size, coder->mf.buffer, @@ -111,6 +110,12 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, coder->mf.write_pos = write_pos; + // Silence Valgrind. lzma_memcmplen() can read extra bytes + // and Valgrind will give warnings if those bytes are uninitialized + // because Valgrind cannot see that the values of the uninitialized + // bytes are eventually ignored. + memzero(coder->mf.buffer + write_pos, LZMA_MEMCMPLEN_EXTRA); + // If end of stream has been reached or flushing completed, we allow // the encoder to process all the input (that is, read_pos is allowed // to reach write_pos). Otherwise we keep keep_size_after bytes @@ -134,7 +139,7 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, && coder->mf.read_pos < coder->mf.read_limit) { // Match finder may update coder->pending and expects it to // start from zero, so use a temporary variable. - const size_t pending = coder->mf.pending; + const uint32_t pending = coder->mf.pending; coder->mf.pending = 0; // Rewind read_pos so that the match finder can hash @@ -152,16 +157,16 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, static lzma_ret -lz_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lz_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_coder *coder = coder_ptr; + while (*out_pos < out_size && (*in_pos < in_size || action != LZMA_RUN)) { - lzma_ret ret; - // Read more data to coder->mf.buffer if needed. if (coder->mf.action == LZMA_RUN && coder->mf.read_pos >= coder->mf.read_limit) @@ -169,7 +174,7 @@ lz_encode(lzma_coder *coder, lzma_allocator *allocator, in, in_pos, in_size, action)); // Encode - ret = coder->lz.code(coder->lz.coder, + const lzma_ret ret = coder->lz.code(coder->lz.coder, &coder->mf, out, out_pos, out_size); if (ret != LZMA_OK) { // Setting this to LZMA_RUN for cases when we are @@ -185,17 +190,9 @@ lz_encode(lzma_coder *coder, lzma_allocator *allocator, static bool -lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, +lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator, const lzma_lz_options *lz_options) { - bool is_bt; - uint32_t new_count; - uint32_t reserve; - uint32_t old_size; - uint32_t hash_bytes; - uint32_t hs; - uint32_t old_count; - // For now, the dictionary size is limited to 1.5 GiB. This may grow // in the future if needed, but it needs a little more work than just // changing this check. @@ -221,14 +218,14 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, // to size_t. // - Memory usage calculation needs something too, e.g. use uint64_t // for mf->size. - reserve = lz_options->dict_size / 2; + uint32_t reserve = lz_options->dict_size / 2; if (reserve > (UINT32_C(1) << 30)) reserve /= 2; reserve += (lz_options->before_size + lz_options->match_len_max + lz_options->after_size) / 2 + (UINT32_C(1) << 19); - old_size = mf->size; + const uint32_t old_size = mf->size; mf->size = mf->keep_size_before + reserve + mf->keep_size_after; // Deallocate the old history buffer if it exists but has different @@ -298,11 +295,12 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, // Calculate the sizes of mf->hash and mf->son and check that // nice_len is big enough for the selected match finder. - hash_bytes = lz_options->match_finder & 0x0F; + const uint32_t hash_bytes = lz_options->match_finder & 0x0F; if (hash_bytes > mf->nice_len) return true; - is_bt = (lz_options->match_finder & 0x10) != 0; + const bool is_bt = (lz_options->match_finder & 0x10) != 0; + uint32_t hs; if (hash_bytes == 2) { hs = 0xFFFF; @@ -338,25 +336,22 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, hs += HASH_4_SIZE; */ - // If the above code calculating hs is modified, make sure that - // this assertion stays valid (UINT32_MAX / 5 is not strictly the - // exact limit). If it doesn't, you need to calculate that - // hash_size_sum + sons_count cannot overflow. - assert(hs < UINT32_MAX / 5); - - old_count = mf->hash_size_sum + mf->sons_count; - mf->hash_size_sum = hs; + const uint32_t old_hash_count = mf->hash_count; + const uint32_t old_sons_count = mf->sons_count; + mf->hash_count = hs; mf->sons_count = mf->cyclic_size; if (is_bt) mf->sons_count *= 2; - new_count = mf->hash_size_sum + mf->sons_count; - // Deallocate the old hash array if it exists and has different size // than what is needed now. - if (old_count != new_count) { + if (old_hash_count != mf->hash_count + || old_sons_count != mf->sons_count) { lzma_free(mf->hash, allocator); mf->hash = NULL; + + lzma_free(mf->son, allocator); + mf->son = NULL; } // Maximum number of match finder cycles @@ -373,16 +368,23 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, static bool -lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator, +lz_encoder_init(lzma_mf *mf, const lzma_allocator *allocator, const lzma_lz_options *lz_options) { - size_t alloc_count; - // Allocate the history buffer. if (mf->buffer == NULL) { - mf->buffer = lzma_alloc(mf->size, allocator); + // lzma_memcmplen() is used for the dictionary buffer + // so we need to allocate a few extra bytes to prevent + // it from reading past the end of the buffer. + mf->buffer = lzma_alloc(mf->size + LZMA_MEMCMPLEN_EXTRA, + allocator); if (mf->buffer == NULL) return true; + + // Keep Valgrind happy with lzma_memcmplen() and initialize + // the extra bytes whose value may get read but which will + // effectively get ignored. + memzero(mf->buffer + mf->size, LZMA_MEMCMPLEN_EXTRA); } // Use cyclic_size as initial mf->offset. This allows @@ -396,43 +398,48 @@ lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator, mf->write_pos = 0; mf->pending = 0; - // Allocate match finder's hash array. - alloc_count = mf->hash_size_sum + mf->sons_count; - #if UINT32_MAX >= SIZE_MAX / 4 // Check for integer overflow. (Huge dictionaries are not // possible on 32-bit CPU.) - if (alloc_count > SIZE_MAX / sizeof(uint32_t)) + if (mf->hash_count > SIZE_MAX / sizeof(uint32_t) + || mf->sons_count > SIZE_MAX / sizeof(uint32_t)) return true; #endif + // Allocate and initialize the hash table. Since EMPTY_HASH_VALUE + // is zero, we can use lzma_alloc_zero() or memzero() for mf->hash. + // + // We don't need to initialize mf->son, but not doing that may + // make Valgrind complain in normalization (see normalize() in + // lz_encoder_mf.c). Skipping the initialization is *very* good + // when big dictionary is used but only small amount of data gets + // actually compressed: most of the mf->son won't get actually + // allocated by the kernel, so we avoid wasting RAM and improve + // initialization speed a lot. if (mf->hash == NULL) { - mf->hash = lzma_alloc(alloc_count * sizeof(uint32_t), + mf->hash = lzma_alloc_zero(mf->hash_count * sizeof(uint32_t), + allocator); + mf->son = lzma_alloc(mf->sons_count * sizeof(uint32_t), allocator); - if (mf->hash == NULL) - return true; - } - mf->son = mf->hash + mf->hash_size_sum; - mf->cyclic_pos = 0; + if (mf->hash == NULL || mf->son == NULL) { + lzma_free(mf->hash, allocator); + mf->hash = NULL; + + lzma_free(mf->son, allocator); + mf->son = NULL; - // Initialize the hash table. Since EMPTY_HASH_VALUE is zero, we - // can use memset(). + return true; + } + } else { /* - for (uint32_t i = 0; i < hash_size_sum; ++i) - mf->hash[i] = EMPTY_HASH_VALUE; + for (uint32_t i = 0; i < mf->hash_count; ++i) + mf->hash[i] = EMPTY_HASH_VALUE; */ - memzero(mf->hash, (size_t)(mf->hash_size_sum) * sizeof(uint32_t)); + memzero(mf->hash, mf->hash_count * sizeof(uint32_t)); + } - // We don't need to initialize mf->son, but not doing that will - // make Valgrind complain in normalization (see normalize() in - // lz_encoder_mf.c). - // - // Skipping this initialization is *very* good when big dictionary is - // used but only small amount of data gets actually compressed: most - // of the mf->hash won't get actually allocated by the kernel, so - // we avoid wasting RAM and improve initialization speed a lot. - //memzero(mf->son, (size_t)(mf->sons_count) * sizeof(uint32_t)); + mf->cyclic_pos = 0; // Handle preset dictionary. if (lz_options->preset_dict != NULL @@ -457,24 +464,32 @@ extern uint64_t lzma_lz_encoder_memusage(const lzma_lz_options *lz_options) { // Old buffers must not exist when calling lz_encoder_prepare(). - lzma_mf mf = { NULL }; + lzma_mf mf = { + .buffer = NULL, + .hash = NULL, + .son = NULL, + .hash_count = 0, + .sons_count = 0, + }; // Setup the size information into mf. if (lz_encoder_prepare(&mf, NULL, lz_options)) return UINT64_MAX; // Calculate the memory usage. - return (uint64_t)(mf.hash_size_sum + mf.sons_count) - * sizeof(uint32_t) - + (uint64_t)(mf.size) + sizeof(lzma_coder); + return ((uint64_t)(mf.hash_count) + mf.sons_count) * sizeof(uint32_t) + + mf.size + sizeof(lzma_coder); } static void -lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +lz_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_coder *coder = coder_ptr; + lzma_next_end(&coder->next, allocator); + lzma_free(coder->mf.son, allocator); lzma_free(coder->mf.hash, allocator); lzma_free(coder->mf.buffer, allocator); @@ -489,10 +504,12 @@ lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lz_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +lz_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_coder *coder = coder_ptr; + if (coder->lz.options_update == NULL) return LZMA_PROG_ERROR; @@ -505,58 +522,63 @@ lz_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_lz_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lz_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_encoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)) { - lzma_lz_options lz_options; - #ifdef HAVE_SMALL // We need that the CRC32 table has been initialized. lzma_crc32_init(); #endif // Allocate and initialize the base data structure. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &lz_encode; next->end = &lz_encoder_end; next->update = &lz_encoder_update; - next->coder->lz.coder = NULL; - next->coder->lz.code = NULL; - next->coder->lz.end = NULL; - - next->coder->mf.buffer = NULL; - next->coder->mf.hash = NULL; - next->coder->mf.hash_size_sum = 0; - next->coder->mf.sons_count = 0; - - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->lz.coder = NULL; + coder->lz.code = NULL; + coder->lz.end = NULL; + + // mf.size is initialized to silence Valgrind + // when used on optimized binaries (GCC may reorder + // code in a way that Valgrind gets unhappy). + coder->mf.buffer = NULL; + coder->mf.size = 0; + coder->mf.hash = NULL; + coder->mf.son = NULL; + coder->mf.hash_count = 0; + coder->mf.sons_count = 0; + + coder->next = LZMA_NEXT_CODER_INIT; } // Initialize the LZ-based encoder. - return_if_error(lz_init(&next->coder->lz, allocator, + lzma_lz_options lz_options; + return_if_error(lz_init(&coder->lz, allocator, filters[0].options, &lz_options)); - // Setup the size information into next->coder->mf and deallocate + // Setup the size information into coder->mf and deallocate // old buffers if they have wrong size. - if (lz_encoder_prepare(&next->coder->mf, allocator, &lz_options)) + if (lz_encoder_prepare(&coder->mf, allocator, &lz_options)) return LZMA_OPTIONS_ERROR; // Allocate new buffers if needed, and do the rest of // the initialization. - if (lz_encoder_init(&next->coder->mf, allocator, &lz_options)) + if (lz_encoder_init(&coder->mf, allocator, &lz_options)) return LZMA_MEM_ERROR; // Initialize the next filter in the chain, if any. - return lzma_next_filter_init(&next->coder->next, allocator, - filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder.h b/Utilities/cmliblzma/liblzma/lz/lz_encoder.h index dcb4b2c..426dcd8 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder.h @@ -119,7 +119,7 @@ struct lzma_mf_s { lzma_action action; /// Number of elements in hash[] - uint32_t hash_size_sum; + uint32_t hash_count; /// Number of elements in son[] uint32_t sons_count; @@ -191,19 +191,18 @@ typedef struct { typedef struct { /// Data specific to the LZ-based encoder - lzma_coder *coder; + void *coder; /// Function to encode from *dict to out[] - lzma_ret (*code)(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size); + lzma_ret (*code)(void *coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); /// Free allocated resources - void (*end)(lzma_coder *coder, lzma_allocator *allocator); + void (*end)(void *coder, const lzma_allocator *allocator); /// Update the options in the middle of the encoding. - lzma_ret (*options_update)(lzma_coder *coder, - const lzma_filter *filter); + lzma_ret (*options_update)(void *coder, const lzma_filter *filter); } lzma_lz_encoder; @@ -218,7 +217,7 @@ typedef struct { /// Get pointer to the first byte not ran through the match finder -static inline uint8_t * +static inline const uint8_t * mf_ptr(const lzma_mf *mf) { return mf->buffer + mf->read_pos; @@ -296,10 +295,10 @@ mf_read(lzma_mf *mf, uint8_t *out, size_t *out_pos, size_t out_size, extern lzma_ret lzma_lz_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_encoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)); diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h index de17c54..342a333 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h @@ -39,22 +39,25 @@ // Endianness doesn't matter in hash_2_calc() (no effect on the output). #ifdef TUKLIB_FAST_UNALIGNED_ACCESS # define hash_2_calc() \ - hash_value = *(const uint16_t *)(cur) + const uint32_t hash_value = *(const uint16_t *)(cur) #else # define hash_2_calc() \ - hash_value = (uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8) + const uint32_t hash_value \ + = (uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8) #endif #define hash_3_calc() \ - temp = hash_table[cur[0]] ^ cur[1]; \ - hash_2_value = temp & HASH_2_MASK; \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8)) & mf->hash_mask + const uint32_t temp = hash_table[cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & mf->hash_mask #define hash_4_calc() \ - temp = hash_table[cur[0]] ^ cur[1]; \ - hash_2_value = temp & HASH_2_MASK; \ - hash_3_value = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ + const uint32_t temp = hash_table[cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ + const uint32_t hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ ^ (hash_table[cur[3]] << 5)) & mf->hash_mask diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c b/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c index 50c3459..7852077 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c @@ -13,6 +13,7 @@ #include "lz_encoder.h" #include "lz_encoder_hash.h" +#include "memcmplen.h" /// \brief Find matches starting from the current byte @@ -32,9 +33,8 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) if (count > 0) { #ifndef NDEBUG - uint32_t i; // Validate the matches. - for (i = 0; i < count; ++i) { + for (uint32_t i = 0; i < count; ++i) { assert(matches[i].len <= mf->nice_len); assert(matches[i].dist < mf->read_pos); assert(memcmp(mf_ptr(mf) - 1, @@ -50,9 +50,6 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) // If a match of maximum search length was found, try to // extend the match to maximum possible length. if (len_best == mf->nice_len) { - uint8_t *p1; - uint8_t *p2; - // The limit for the match length is either the // maximum match length supported by the LZ-based // encoder or the number of bytes left in the @@ -63,15 +60,13 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) // Pointer to the byte we just ran through // the match finder. - p1 = mf_ptr(mf) - 1; + const uint8_t *p1 = mf_ptr(mf) - 1; // Pointer to the beginning of the match. We need -1 // here because the match distances are zero based. - p2 = p1 - matches[count - 1].dist - 1; + const uint8_t *p2 = p1 - matches[count - 1].dist - 1; - while (len_best < limit - && p1[len_best] == p2[len_best]) - ++len_best; + len_best = lzma_memcmplen(p1, p2, len_best, limit); } } @@ -112,36 +107,35 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) static void normalize(lzma_mf *mf) { - uint32_t i; - uint32_t subvalue; - uint32_t count; - uint32_t *hash; - assert(mf->read_pos + mf->offset == MUST_NORMALIZE_POS); // In future we may not want to touch the lowest bits, because there // may be match finders that use larger resolution than one byte. - subvalue = (MUST_NORMALIZE_POS - mf->cyclic_size); + const uint32_t subvalue + = (MUST_NORMALIZE_POS - mf->cyclic_size); // & (~(UINT32_C(1) << 10) - 1); - count = mf->hash_size_sum + mf->sons_count; - hash = mf->hash; - - for (i = 0; i < count; ++i) { + for (uint32_t i = 0; i < mf->hash_count; ++i) { // If the distance is greater than the dictionary size, // we can simply mark the hash element as empty. + if (mf->hash[i] <= subvalue) + mf->hash[i] = EMPTY_HASH_VALUE; + else + mf->hash[i] -= subvalue; + } + + for (uint32_t i = 0; i < mf->sons_count; ++i) { + // Do the same for mf->son. // - // NOTE: Only the first mf->hash_size_sum elements are - // initialized for sure. There may be uninitialized elements - // in mf->son. Since we go through both mf->hash and - // mf->son here in normalization, Valgrind may complain - // that the "if" below depends on uninitialized value. In - // this case it is safe to ignore the warning. See also the - // comments in lz_encoder_init() in lz_encoder.c. - if (hash[i] <= subvalue) - hash[i] = EMPTY_HASH_VALUE; + // NOTE: There may be uninitialized elements in mf->son. + // Valgrind may complain that the "if" below depends on + // an uninitialized value. In this case it is safe to ignore + // the warning. See also the comments in lz_encoder_init() + // in lz_encoder.c. + if (mf->son[i] <= subvalue) + mf->son[i] = EMPTY_HASH_VALUE; else - hash[i] -= subvalue; + mf->son[i] -= subvalue; } // Update offset to match the new locations. @@ -204,14 +198,15 @@ move_pending(lzma_mf *mf) move_pending(mf); \ ret_op; \ } \ - cur = mf_ptr(mf); \ - pos = mf->read_pos + mf->offset + const uint8_t *cur = mf_ptr(mf); \ + const uint32_t pos = mf->read_pos + mf->offset /// Header for find functions. "return 0" indicates that zero matches /// were found. #define header_find(is_bt, len_min) \ - header(is_bt, len_min, return 0) + header(is_bt, len_min, return 0); \ + uint32_t matches_count = 0 /// Header for a loop in a skip function. "continue" tells to skip the rest @@ -268,19 +263,15 @@ hc_find_func( while (true) { const uint32_t delta = pos - cur_match; - const uint8_t *pb; if (depth-- == 0 || delta >= cyclic_size) return matches; - pb = cur - delta; + const uint8_t *const pb = cur - delta; cur_match = son[cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)]; if (pb[len_best] == cur[len_best] && pb[0] == cur[0]) { - uint32_t len = 0; - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + uint32_t len = lzma_memcmplen(pb, cur, 1, len_limit); if (len_best < len) { len_best = len; @@ -313,27 +304,21 @@ do { \ extern uint32_t lzma_mf_hc3_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t delta2, cur_match; - uint32_t len_best = 2; - uint32_t matches_count = 0; - header_find(false, 3); hash_3_calc(); - delta2 = pos - mf->hash[hash_2_value]; - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 2; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen(cur - delta2, cur, + len_best, len_limit); matches[0].len = len_best; matches[0].dist = delta2 - 1; @@ -353,22 +338,18 @@ extern void lzma_mf_hc3_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t cur_match; - if (mf_avail(mf) < 3) { move_pending(mf); continue; } - cur = mf_ptr(mf); - pos = mf->read_pos + mf->offset; + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; hash_3_calc(); - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; @@ -384,25 +365,21 @@ lzma_mf_hc3_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t delta2, delta3, cur_match; - uint32_t len_best = 1; - uint32_t matches_count = 0; - header_find(false, 4); hash_4_calc(); - delta2 = pos - mf->hash[hash_2_value]; - delta3 = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value ] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 1; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { len_best = 2; matches[0].len = 2; @@ -418,9 +395,8 @@ lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches) } if (matches_count != 0) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen(cur - delta2, cur, + len_best, len_limit); matches[matches_count - 1].len = len_best; @@ -441,22 +417,18 @@ extern void lzma_mf_hc4_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t cur_match; - if (mf_avail(mf) < 4) { move_pending(mf); continue; } - cur = mf_ptr(mf); - pos = mf->read_pos + mf->offset; + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; hash_4_calc(); - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; @@ -494,10 +466,6 @@ bt_find_func( uint32_t len1 = 0; while (true) { - uint32_t *pair; - const uint8_t *pb; - uint32_t len; - const uint32_t delta = pos - cur_match; if (depth-- == 0 || delta >= cyclic_size) { *ptr0 = EMPTY_HASH_VALUE; @@ -505,17 +473,15 @@ bt_find_func( return matches; } - pair = son + ((cyclic_pos - delta + uint32_t *const pair = son + ((cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)) << 1); - pb = cur - delta; - len = my_min(len0, len1); + const uint8_t *const pb = cur - delta; + uint32_t len = my_min(len0, len1); if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + len = lzma_memcmplen(pb, cur, len + 1, len_limit); if (len_best < len) { len_best = len; @@ -564,10 +530,6 @@ bt_skip_func( uint32_t len1 = 0; while (true) { - uint32_t *pair; - const uint8_t *pb; - uint32_t len; - const uint32_t delta = pos - cur_match; if (depth-- == 0 || delta >= cyclic_size) { *ptr0 = EMPTY_HASH_VALUE; @@ -575,16 +537,14 @@ bt_skip_func( return; } - pair = son + ((cyclic_pos - delta + uint32_t *pair = son + ((cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)) << 1); - pb = cur - delta; - len = my_min(len0, len1); + const uint8_t *pb = cur - delta; + uint32_t len = my_min(len0, len1); if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + len = lzma_memcmplen(pb, cur, len + 1, len_limit); if (len == len_limit) { *ptr1 = pair[0]; @@ -626,17 +586,11 @@ do { \ extern uint32_t lzma_mf_bt2_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t hash_value; /* hash_2_calc */ - uint32_t cur_match; - uint32_t matches_count = 0; - header_find(true, 2); hash_2_calc(); - cur_match = mf->hash[hash_value]; + const uint32_t cur_match = mf->hash[hash_value]; mf->hash[hash_value] = pos; bt_find(1); @@ -647,16 +601,11 @@ extern void lzma_mf_bt2_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t hash_value; /* hash_2_calc */ - uint32_t cur_match; - header_skip(true, 2); hash_2_calc(); - cur_match = mf->hash[hash_value]; + const uint32_t cur_match = mf->hash[hash_value]; mf->hash[hash_value] = pos; bt_skip(); @@ -670,27 +619,21 @@ lzma_mf_bt2_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_bt3_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t delta2, cur_match; - uint32_t len_best = 2; - uint32_t matches_count = 0; - header_find(true, 3); hash_3_calc(); - delta2 = pos - mf->hash[hash_2_value]; - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 2; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen( + cur, cur - delta2, len_best, len_limit); matches[0].len = len_best; matches[0].dist = delta2 - 1; @@ -710,16 +653,12 @@ extern void lzma_mf_bt3_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t cur_match; - header_skip(true, 3); hash_3_calc(); - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; @@ -735,25 +674,21 @@ lzma_mf_bt3_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t delta2, delta3, cur_match; - uint32_t len_best = 1; - uint32_t matches_count = 0; - header_find(true, 4); hash_4_calc(); - delta2 = pos - mf->hash[hash_2_value]; - delta3 = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 1; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { len_best = 2; matches[0].len = 2; @@ -769,9 +704,8 @@ lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches) } if (matches_count != 0) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen( + cur, cur - delta2, len_best, len_limit); matches[matches_count - 1].len = len_best; @@ -792,16 +726,12 @@ extern void lzma_mf_bt4_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t cur_match; - header_skip(true, 4); hash_4_calc(); - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; diff --git a/Utilities/cmliblzma/liblzma/lzma/fastpos.h b/Utilities/cmliblzma/liblzma/lzma/fastpos.h index 5a834d6..a3feea5 100644 --- a/Utilities/cmliblzma/liblzma/lzma/fastpos.h +++ b/Utilities/cmliblzma/liblzma/lzma/fastpos.h @@ -14,15 +14,15 @@ #ifndef LZMA_FASTPOS_H #define LZMA_FASTPOS_H -// LZMA encodes match distances (positions) by storing the highest two -// bits using a six-bit value [0, 63], and then the missing lower bits. -// Dictionary size is also stored using this encoding in the new .lzma +// LZMA encodes match distances by storing the highest two bits using +// a six-bit value [0, 63], and then the missing lower bits. +// Dictionary size is also stored using this encoding in the .xz // file format header. // // fastpos.h provides a way to quickly find out the correct six-bit // values. The following table gives some examples of this encoding: // -// pos return +// dist return // 0 0 // 1 1 // 2 2 @@ -48,10 +48,10 @@ // Provided functions or macros // ---------------------------- // -// get_pos_slot(pos) is the basic version. get_pos_slot_2(pos) -// assumes that pos >= FULL_DISTANCES, thus the result is at least -// FULL_DISTANCES_BITS * 2. Using get_pos_slot(pos) instead of -// get_pos_slot_2(pos) would give the same result, but get_pos_slot_2(pos) +// get_dist_slot(dist) is the basic version. get_dist_slot_2(dist) +// assumes that dist >= FULL_DISTANCES, thus the result is at least +// FULL_DISTANCES_BITS * 2. Using get_dist_slot(dist) instead of +// get_dist_slot_2(dist) would give the same result, but get_dist_slot_2(dist) // should be tiny bit faster due to the assumption being made. // // @@ -75,16 +75,15 @@ // on all systems I have tried. The size optimized version is sometimes // slightly faster, but sometimes it is a lot slower. -#include "config.h" - #ifdef HAVE_SMALL -# define get_pos_slot(pos) ((pos) <= 4 ? (pos) : get_pos_slot_2(pos)) +# define get_dist_slot(dist) \ + ((dist) <= 4 ? (dist) : get_dist_slot_2(dist)) static inline uint32_t -get_pos_slot_2(uint32_t pos) +get_dist_slot_2(uint32_t dist) { - const uint32_t i = bsr32(pos); - return (i + i) + ((pos >> (i - 1)) & 1); + const uint32_t i = bsr32(dist); + return (i + i) + ((dist >> (i - 1)) & 1); } @@ -101,39 +100,39 @@ extern const uint8_t lzma_fastpos[1 << FASTPOS_BITS]; #define fastpos_limit(extra, n) \ (UINT32_C(1) << (FASTPOS_BITS + fastpos_shift(extra, n))) -#define fastpos_result(pos, extra, n) \ - lzma_fastpos[(pos) >> fastpos_shift(extra, n)] \ +#define fastpos_result(dist, extra, n) \ + lzma_fastpos[(dist) >> fastpos_shift(extra, n)] \ + 2 * fastpos_shift(extra, n) static inline uint32_t -get_pos_slot(uint32_t pos) +get_dist_slot(uint32_t dist) { // If it is small enough, we can pick the result directly from // the precalculated table. - if (pos < fastpos_limit(0, 0)) - return lzma_fastpos[pos]; + if (dist < fastpos_limit(0, 0)) + return lzma_fastpos[dist]; - if (pos < fastpos_limit(0, 1)) - return fastpos_result(pos, 0, 1); + if (dist < fastpos_limit(0, 1)) + return fastpos_result(dist, 0, 1); - return fastpos_result(pos, 0, 2); + return fastpos_result(dist, 0, 2); } #ifdef FULL_DISTANCES_BITS static inline uint32_t -get_pos_slot_2(uint32_t pos) +get_dist_slot_2(uint32_t dist) { - assert(pos >= FULL_DISTANCES); + assert(dist >= FULL_DISTANCES); - if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 0)) - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 0); + if (dist < fastpos_limit(FULL_DISTANCES_BITS - 1, 0)) + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 0); - if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 1)) - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 1); + if (dist < fastpos_limit(FULL_DISTANCES_BITS - 1, 1)) + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 1); - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 2); + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 2); } #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c index bd2a737..878c870 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c @@ -16,7 +16,7 @@ #include "lzma_decoder.h" -struct lzma_coder_s { +typedef struct { enum sequence { SEQ_CONTROL, SEQ_UNCOMPRESSED_1, @@ -50,14 +50,16 @@ struct lzma_coder_s { bool need_dictionary_reset; lzma_options_lzma options; -}; +} lzma_lzma2_coder; static lzma_ret -lzma2_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dict, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lzma2_decode(void *coder_ptr, lzma_dict *restrict dict, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size) { + lzma_lzma2_coder *restrict coder = coder_ptr; + // With SEQ_LZMA it is possible that no new input is needed to do // some progress. The rest of the sequences assume that there is // at least one byte of input. @@ -209,8 +211,10 @@ lzma2_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dict, static void -lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +lzma2_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_lzma2_coder *coder = coder_ptr; + assert(coder->lzma.end == NULL); lzma_free(coder->lzma.coder, allocator); @@ -221,34 +225,36 @@ lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lzma2_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma2_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options) { - const lzma_options_lzma *options = opt; - - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (lz->coder == NULL) + lzma_lzma2_coder *coder = lz->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_lzma2_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + lz->coder = coder; lz->code = &lzma2_decode; lz->end = &lzma2_decoder_end; - lz->coder->lzma = LZMA_LZ_DECODER_INIT; + coder->lzma = LZMA_LZ_DECODER_INIT; } - lz->coder->sequence = SEQ_CONTROL; - lz->coder->need_properties = true; - lz->coder->need_dictionary_reset = options->preset_dict == NULL + const lzma_options_lzma *options = opt; + + coder->sequence = SEQ_CONTROL; + coder->need_properties = true; + coder->need_dictionary_reset = options->preset_dict == NULL || options->preset_dict_size == 0; - return lzma_lzma_decoder_create(&lz->coder->lzma, + return lzma_lzma_decoder_create(&coder->lzma, allocator, options, lz_options); } extern lzma_ret -lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma2_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { // LZMA2 can only be the last filter in the chain. This is enforced @@ -263,17 +269,15 @@ lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern uint64_t lzma_lzma2_decoder_memusage(const void *options) { - return sizeof(lzma_coder) + return sizeof(lzma_lzma2_coder) + lzma_lzma_decoder_memusage_nocheck(options); } extern lzma_ret -lzma_lzma2_props_decode(void **options, lzma_allocator *allocator, +lzma_lzma2_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_lzma *opt; - if (props_size != 1) return LZMA_OPTIONS_ERROR; @@ -285,7 +289,8 @@ lzma_lzma2_props_decode(void **options, lzma_allocator *allocator, if (props[0] > 40) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_lzma), allocator); + lzma_options_lzma *opt = lzma_alloc( + sizeof(lzma_options_lzma), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h index fac4ac4..ef2dcbf 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h @@ -17,12 +17,13 @@ #include "common.h" extern lzma_ret lzma_lzma2_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma2_decoder_memusage(const void *options); extern lzma_ret lzma_lzma2_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c index a3651a7..63588ee 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c @@ -17,7 +17,7 @@ #include "lzma2_encoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INIT, SEQ_LZMA_ENCODE, @@ -27,7 +27,7 @@ struct lzma_coder_s { } sequence; /// LZMA encoder - lzma_coder *lzma; + void *lzma; /// LZMA options currently in use. lzma_options_lzma opt_cur; @@ -48,20 +48,19 @@ struct lzma_coder_s { /// Buffer to hold the chunk header and LZMA compressed data uint8_t buf[LZMA2_HEADER_MAX + LZMA2_CHUNK_MAX]; -}; +} lzma_lzma2_coder; static void -lzma2_header_lzma(lzma_coder *coder) +lzma2_header_lzma(lzma_lzma2_coder *coder) { - size_t pos; - size_t size; - assert(coder->uncompressed_size > 0); assert(coder->uncompressed_size <= LZMA2_UNCOMPRESSED_MAX); assert(coder->compressed_size > 0); assert(coder->compressed_size <= LZMA2_CHUNK_MAX); + size_t pos; + if (coder->need_properties) { pos = 0; @@ -82,7 +81,7 @@ lzma2_header_lzma(lzma_coder *coder) coder->buf_pos = pos; // Uncompressed size - size = coder->uncompressed_size - 1; + size_t size = coder->uncompressed_size - 1; coder->buf[pos++] += size >> 16; coder->buf[pos++] = (size >> 8) & 0xFF; coder->buf[pos++] = size & 0xFF; @@ -109,7 +108,7 @@ lzma2_header_lzma(lzma_coder *coder) static void -lzma2_header_uncompressed(lzma_coder *coder) +lzma2_header_uncompressed(lzma_lzma2_coder *coder) { assert(coder->uncompressed_size > 0); assert(coder->uncompressed_size <= LZMA2_CHUNK_MAX); @@ -134,10 +133,12 @@ lzma2_header_uncompressed(lzma_coder *coder) static lzma_ret -lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma2_encode(void *coder_ptr, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { + lzma_lzma2_coder *restrict coder = coder_ptr; + while (*out_pos < out_size) switch (coder->sequence) { case SEQ_INIT: @@ -163,9 +164,6 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Fall through case SEQ_LZMA_ENCODE: { - uint32_t read_start; - lzma_ret ret; - // Calculate how much more uncompressed data this chunk // could accept. const uint32_t left = LZMA2_UNCOMPRESSED_MAX @@ -186,10 +184,10 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Save the start position so that we can update // coder->uncompressed_size. - read_start = mf->read_pos - mf->read_ahead; + const uint32_t read_start = mf->read_pos - mf->read_ahead; // Call the LZMA encoder until the chunk is finished. - ret = lzma_lzma_encode(coder->lzma, mf, + const lzma_ret ret = lzma_lzma_encode(coder->lzma, mf, coder->buf + LZMA2_HEADER_MAX, &coder->compressed_size, LZMA2_CHUNK_MAX, limit); @@ -266,8 +264,9 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static void -lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +lzma2_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_lzma2_coder *coder = coder_ptr; lzma_free(coder->lzma, allocator); lzma_free(coder, allocator); return; @@ -275,9 +274,9 @@ lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) +lzma2_encoder_options_update(void *coder_ptr, const lzma_filter *filter) { - lzma_options_lzma *opt; + lzma_lzma2_coder *coder = coder_ptr; // New options can be set only when there is no incomplete chunk. // This is the case at the beginning of the raw stream and right @@ -287,7 +286,7 @@ lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) // Look if there are new options. At least for now, // only lc/lp/pb can be changed. - opt = filter->options; + const lzma_options_lzma *opt = filter->options; if (coder->opt_cur.lc != opt->lc || coder->opt_cur.lp != opt->lp || coder->opt_cur.pb != opt->pb) { // Validate the options. @@ -310,36 +309,38 @@ lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) static lzma_ret -lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, +lzma2_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { if (options == NULL) return LZMA_PROG_ERROR; - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (lz->coder == NULL) + lzma_lzma2_coder *coder = lz->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_lzma2_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + lz->coder = coder; lz->code = &lzma2_encode; lz->end = &lzma2_encoder_end; lz->options_update = &lzma2_encoder_options_update; - lz->coder->lzma = NULL; + coder->lzma = NULL; } - lz->coder->opt_cur = *(const lzma_options_lzma *)(options); + coder->opt_cur = *(const lzma_options_lzma *)(options); - lz->coder->sequence = SEQ_INIT; - lz->coder->need_properties = true; - lz->coder->need_state_reset = false; - lz->coder->need_dictionary_reset - = lz->coder->opt_cur.preset_dict == NULL - || lz->coder->opt_cur.preset_dict_size == 0; + coder->sequence = SEQ_INIT; + coder->need_properties = true; + coder->need_state_reset = false; + coder->need_dictionary_reset + = coder->opt_cur.preset_dict == NULL + || coder->opt_cur.preset_dict_size == 0; // Initialize LZMA encoder - return_if_error(lzma_lzma_encoder_create(&lz->coder->lzma, allocator, - &lz->coder->opt_cur, lz_options)); + return_if_error(lzma_lzma_encoder_create(&coder->lzma, allocator, + &coder->opt_cur, lz_options)); // Make sure that we will always have enough history available in // case we need to use uncompressed chunks. They are used when the @@ -355,7 +356,7 @@ lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma2_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma2_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { return lzma_lz_encoder_init( @@ -370,7 +371,7 @@ lzma_lzma2_encoder_memusage(const void *options) if (lzma_mem == UINT64_MAX) return UINT64_MAX; - return sizeof(lzma_coder) + lzma_mem; + return sizeof(lzma_lzma2_coder) + lzma_mem; } @@ -393,7 +394,17 @@ lzma_lzma2_props_encode(const void *options, uint8_t *out) if (d == UINT32_MAX) out[0] = 40; else - out[0] = get_pos_slot(d + 1) - 24; + out[0] = get_dist_slot(d + 1) - 24; return LZMA_OK; } + + +extern uint64_t +lzma_lzma2_block_size(const void *options) +{ + const lzma_options_lzma *const opt = options; + + // Use at least 1 MiB to keep compression ratio better. + return my_max((uint64_t)(opt->dict_size) * 3, UINT64_C(1) << 20); +} diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h index ca19ef4..515f183 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h @@ -31,11 +31,13 @@ extern lzma_ret lzma_lzma2_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); extern uint64_t lzma_lzma2_encoder_memusage(const void *options); extern lzma_ret lzma_lzma2_props_encode(const void *options, uint8_t *out); +extern uint64_t lzma_lzma2_block_size(const void *options); + #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_common.h b/Utilities/cmliblzma/liblzma/lzma/lzma_common.h index 36267dc..09efd38 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_common.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_common.h @@ -129,15 +129,12 @@ static inline void literal_init(probability (*probs)[LITERAL_CODER_SIZE], uint32_t lc, uint32_t lp) { - uint32_t coders; - uint32_t i, j; - assert(lc + lp <= LZMA_LCLP_MAX); - coders = 1U << (lc + lp); + const uint32_t coders = 1U << (lc + lp); - for (i = 0; i < coders; ++i) - for (j = 0; j < LITERAL_CODER_SIZE; ++j) + for (uint32_t i = 0; i < coders; ++i) + for (uint32_t j = 0; j < LITERAL_CODER_SIZE; ++j) bit_reset(probs[i][j]); return; @@ -174,53 +171,54 @@ literal_init(probability (*probs)[LITERAL_CODER_SIZE], // Match distance // //////////////////// -// Different set of probabilities is used for match distances that have very +// Different sets of probabilities are used for match distances that have very // short match length: Lengths of 2, 3, and 4 bytes have a separate set of // probabilities for each length. The matches with longer length use a shared // set of probabilities. -#define LEN_TO_POS_STATES 4 +#define DIST_STATES 4 // Macro to get the index of the appropriate probability array. -#define get_len_to_pos_state(len) \ - ((len) < LEN_TO_POS_STATES + MATCH_LEN_MIN \ +#define get_dist_state(len) \ + ((len) < DIST_STATES + MATCH_LEN_MIN \ ? (len) - MATCH_LEN_MIN \ - : LEN_TO_POS_STATES - 1) + : DIST_STATES - 1) -// The highest two bits of a match distance (pos slot) are encoded using six -// bits. See fastpos.h for more explanation. -#define POS_SLOT_BITS 6 -#define POS_SLOTS (1 << POS_SLOT_BITS) +// The highest two bits of a match distance (distance slot) are encoded +// using six bits. See fastpos.h for more explanation. +#define DIST_SLOT_BITS 6 +#define DIST_SLOTS (1 << DIST_SLOT_BITS) // Match distances up to 127 are fully encoded using probabilities. Since -// the highest two bits (pos slot) are always encoded using six bits, the -// distances 0-3 don't need any additional bits to encode, since the pos -// slot itself is the same as the actual distance. START_POS_MODEL_INDEX -// indicates the first pos slot where at least one additional bit is needed. -#define START_POS_MODEL_INDEX 4 +// the highest two bits (distance slot) are always encoded using six bits, +// the distances 0-3 don't need any additional bits to encode, since the +// distance slot itself is the same as the actual distance. DIST_MODEL_START +// indicates the first distance slot where at least one additional bit is +// needed. +#define DIST_MODEL_START 4 // Match distances greater than 127 are encoded in three pieces: -// - pos slot: the highest two bits +// - distance slot: the highest two bits // - direct bits: 2-26 bits below the highest two bits // - alignment bits: four lowest bits // // Direct bits don't use any probabilities. // -// The pos slot value of 14 is for distances 128-191 (see the table in +// The distance slot value of 14 is for distances 128-191 (see the table in // fastpos.h to understand why). -#define END_POS_MODEL_INDEX 14 +#define DIST_MODEL_END 14 -// Pos slots that indicate a distance <= 127. -#define FULL_DISTANCES_BITS (END_POS_MODEL_INDEX / 2) +// Distance slots that indicate a distance <= 127. +#define FULL_DISTANCES_BITS (DIST_MODEL_END / 2) #define FULL_DISTANCES (1 << FULL_DISTANCES_BITS) // For match distances greater than 127, only the highest two bits and the // lowest four bits (alignment) is encoded using probabilities. #define ALIGN_BITS 4 -#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS) -#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1) +#define ALIGN_SIZE (1 << ALIGN_BITS) +#define ALIGN_MASK (ALIGN_SIZE - 1) // LZMA remembers the four most recent match distances. Reusing these distances // tends to take less space than re-encoding the actual distance value. -#define REP_DISTANCES 4 +#define REPS 4 #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c index 3c0f393..d0f29b7 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c @@ -16,6 +16,12 @@ #include "lzma_decoder.h" #include "range_decoder.h" +// The macros unroll loops with switch statements. +// Silence warnings about missing fall-through comments. +#if TUKLIB_GNUC_REQ(7, 0) +# pragma GCC diagnostic ignored "-Wimplicit-fallthrough" +#endif + #ifdef HAVE_SMALL @@ -114,33 +120,33 @@ do { \ case seq ## _CHOICE: \ rc_if_0(ld.choice, seq ## _CHOICE) { \ rc_update_0(ld.choice); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW0); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW1); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW2); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW0); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW1); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW2); \ target = symbol - LEN_LOW_SYMBOLS + MATCH_LEN_MIN; \ } else { \ rc_update_1(ld.choice); \ case seq ## _CHOICE2: \ rc_if_0(ld.choice2, seq ## _CHOICE2) { \ rc_update_0(ld.choice2); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID0); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID1); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID2); \ target = symbol - LEN_MID_SYMBOLS \ + MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \ } else { \ rc_update_1(ld.choice2); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH0); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH1); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH2); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH3); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH4); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH5); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH6); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH7); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH0); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH1); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH2); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH3); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH4); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH5); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH6); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH7); \ target = symbol - LEN_HIGH_SYMBOLS \ + MATCH_LEN_MIN \ + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ @@ -161,7 +167,7 @@ typedef struct { } lzma_length_decoder; -struct lzma_coder_s { +typedef struct { /////////////////// // Probabilities // /////////////////// @@ -193,15 +199,15 @@ struct lzma_coder_s { /// Probability tree for the highest two bits of the match distance. /// There is a separate probability tree for match lengths of /// 2 (i.e. MATCH_LEN_MIN), 3, 4, and [5, 273]. - probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; + probability dist_slot[DIST_STATES][DIST_SLOTS]; /// Probability trees for additional bits for match distance when the /// distance is in the range [4, 127]. - probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; + probability pos_special[FULL_DISTANCES - DIST_MODEL_END]; /// Probability tree for the lowest four bits of a match distance /// that is equal to or greater than 128. - probability pos_align[ALIGN_TABLE_SIZE]; + probability pos_align[ALIGN_SIZE]; /// Length of a normal match lzma_length_decoder match_len_decoder; @@ -245,8 +251,8 @@ struct lzma_coder_s { SEQ_LITERAL_WRITE, SEQ_IS_REP, seq_len(SEQ_MATCH_LEN), - seq_6(SEQ_POS_SLOT), - SEQ_POS_MODEL, + seq_6(SEQ_DIST_SLOT), + SEQ_DIST_MODEL, SEQ_DIRECT, seq_4(SEQ_ALIGN), SEQ_EOPM, @@ -277,14 +283,27 @@ struct lzma_coder_s { /// If decoding a literal: match byte. /// If decoding a match: length of the match. uint32_t len; -}; +} lzma_lzma1_decoder; static lzma_ret -lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, - const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size) +lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, + const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) { + lzma_lzma1_decoder *restrict coder = coder_ptr; + + //////////////////// + // Initialization // + //////////////////// + + { + const lzma_ret ret = rc_read_init( + &coder->rc, in, in_pos, in_size); + if (ret != LZMA_STREAM_END) + return ret; + } + /////////////// // Variables // /////////////// @@ -331,16 +350,6 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, if (no_eopm && coder->uncompressed_size < dict.limit - dict.pos) dict.limit = dict.pos + (size_t)(coder->uncompressed_size); - //////////////////// - // Initialization // - //////////////////// - - if (!rc_read_init(&coder->rc, in, in_pos, in_size)) - return LZMA_OK; - - rc = coder->rc; - rc_in_pos = *in_pos; - // The main decoder loop. The "switch" is used to restart the decoder at // correct location. Once restarted, the "switch" is no longer used. switch (coder->sequence) @@ -356,21 +365,6 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, break; rc_if_0(coder->is_match[state][pos_state], SEQ_IS_MATCH) { - static const lzma_lzma_state next_state[] = { - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_MATCH_LIT_LIT, - STATE_REP_LIT_LIT, - STATE_SHORTREP_LIT_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT, - STATE_SHORTREP_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT - }; - rc_update_0(coder->is_match[state][pos_state]); // It's a literal i.e. a single 8-bit byte. @@ -388,21 +382,16 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, rc_bit(probs[symbol], , , SEQ_LITERAL); } while (symbol < (1 << 8)); #else - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL0); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL1); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL2); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL3); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL4); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL5); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL6); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL7); + rc_bit_case(probs[symbol], , , SEQ_LITERAL0); + rc_bit_case(probs[symbol], , , SEQ_LITERAL1); + rc_bit_case(probs[symbol], , , SEQ_LITERAL2); + rc_bit_case(probs[symbol], , , SEQ_LITERAL3); + rc_bit_case(probs[symbol], , , SEQ_LITERAL4); + rc_bit_case(probs[symbol], , , SEQ_LITERAL5); + rc_bit_case(probs[symbol], , , SEQ_LITERAL6); + rc_bit_case(probs[symbol], , , SEQ_LITERAL7); #endif } else { -#ifndef HAVE_SMALL - uint32_t match_bit; - uint32_t subcoder_index; -#endif - // Decode literal with match byte. // // We store the byte we compare against @@ -441,6 +430,8 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, } while (symbol < (1 << 8)); #else // Unroll the loop. + uint32_t match_bit; + uint32_t subcoder_index; # define d(seq) \ case seq: \ @@ -474,6 +465,20 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // Use a lookup table to update to literal state, // since compared to other state updates, this would // need two branches. + static const lzma_lzma_state next_state[] = { + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_MATCH_LIT_LIT, + STATE_REP_LIT_LIT, + STATE_SHORTREP_LIT_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT, + STATE_SHORTREP_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT + }; state = next_state[state]; case SEQ_LITERAL_WRITE: @@ -509,28 +514,28 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // Prepare to decode the highest two bits of the // match distance. - probs = coder->pos_slot[get_len_to_pos_state(len)]; + probs = coder->dist_slot[get_dist_state(len)]; symbol = 1; #ifdef HAVE_SMALL - case SEQ_POS_SLOT: + case SEQ_DIST_SLOT: do { - rc_bit(probs[symbol], , , SEQ_POS_SLOT); - } while (symbol < POS_SLOTS); + rc_bit(probs[symbol], , , SEQ_DIST_SLOT); + } while (symbol < DIST_SLOTS); #else - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT0); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT1); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT2); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT3); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT4); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT5); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT0); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT1); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT2); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT3); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT4); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT5); #endif // Get rid of the highest bit that was needed for // indexing of the probability array. - symbol -= POS_SLOTS; + symbol -= DIST_SLOTS; assert(symbol <= 63); - if (symbol < START_POS_MODEL_INDEX) { + if (symbol < DIST_MODEL_START) { // Match distances [0, 3] have only two bits. rep0 = symbol; } else { @@ -540,7 +545,7 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, assert(limit >= 1 && limit <= 30); rep0 = 2 + (symbol & 1); - if (symbol < END_POS_MODEL_INDEX) { + if (symbol < DIST_MODEL_END) { // Prepare to decode the low bits for // a distance of [4, 127]. assert(limit <= 5); @@ -560,38 +565,38 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, - symbol - 1; symbol = 1; offset = 0; - case SEQ_POS_MODEL: + case SEQ_DIST_MODEL: #ifdef HAVE_SMALL do { rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); } while (++offset < limit); #else switch (limit) { case 5: assert(offset == 0); - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 4: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 3: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 2: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 1: @@ -601,9 +606,9 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // rc_bit_last() here to omit // the unneeded updating of // "symbol". - rc_bit_last(probs[symbol], 0, + rc_bit_last(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); } #endif } else { @@ -635,19 +640,19 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, } while (++offset < ALIGN_BITS); #else case SEQ_ALIGN0: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 1, SEQ_ALIGN0); case SEQ_ALIGN1: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 2, SEQ_ALIGN1); case SEQ_ALIGN2: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 4, SEQ_ALIGN2); case SEQ_ALIGN3: - // Like in SEQ_POS_MODEL, we don't + // Like in SEQ_DIST_MODEL, we don't // need "symbol" for anything else // than indexing the probability array. - rc_bit_last(coder->pos_align[symbol], 0, + rc_bit_last(coder->pos_align[symbol], , rep0 += 8, SEQ_ALIGN3); #endif @@ -732,11 +737,9 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // is stored to rep0 and rep1, rep2 and rep3 // are updated accordingly. rc_if_0(coder->is_rep1[state], SEQ_IS_REP1) { - uint32_t distance; - rc_update_0(coder->is_rep1[state]); - distance = rep1; + const uint32_t distance = rep1; rep1 = rep0; rep0 = distance; @@ -745,23 +748,19 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, case SEQ_IS_REP2: rc_if_0(coder->is_rep2[state], SEQ_IS_REP2) { - uint32_t distance; - rc_update_0(coder->is_rep2[ state]); - distance = rep2; + const uint32_t distance = rep2; rep2 = rep1; rep1 = rep0; rep0 = distance; } else { - uint32_t distance; - rc_update_1(coder->is_rep2[ state]); - distance = rep3; + const uint32_t distance = rep3; rep3 = rep2; rep2 = rep1; rep1 = rep0; @@ -849,26 +848,17 @@ out: static void -lzma_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) +lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) { + lzma_lzma1_decoder *coder = coder_ptr; coder->uncompressed_size = uncompressed_size; } -/* -extern void -lzma_lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) -{ - // This is hack. - (*(lzma_coder **)(coder))->uncompressed_size = uncompressed_size; -} -*/ static void -lzma_decoder_reset(lzma_coder *coder, const void *opt) +lzma_decoder_reset(void *coder_ptr, const void *opt) { - uint32_t i, j, pos_state; - uint32_t num_pos_states; - + lzma_lzma1_decoder *coder = coder_ptr; const lzma_options_lzma *options = opt; // NOTE: We assume that lc/lp/pb are valid since they were @@ -895,8 +885,8 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) rc_reset(coder->rc); // Bit and bittree decoders - for (i = 0; i < STATES; ++i) { - for (j = 0; j <= coder->pos_mask; ++j) { + for (uint32_t i = 0; i < STATES; ++i) { + for (uint32_t j = 0; j <= coder->pos_mask; ++j) { bit_reset(coder->is_match[i][j]); bit_reset(coder->is_rep0_long[i][j]); } @@ -907,22 +897,22 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) bit_reset(coder->is_rep2[i]); } - for (i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); + for (uint32_t i = 0; i < DIST_STATES; ++i) + bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS); - for (i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) + for (uint32_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i) bit_reset(coder->pos_special[i]); bittree_reset(coder->pos_align, ALIGN_BITS); // Len decoders (also bit/bittree) - num_pos_states = 1U << options->pb; + const uint32_t num_pos_states = 1U << options->pb; bit_reset(coder->match_len_decoder.choice); bit_reset(coder->match_len_decoder.choice2); bit_reset(coder->rep_len_decoder.choice); bit_reset(coder->rep_len_decoder.choice2); - for (pos_state = 0; pos_state < num_pos_states; ++pos_state) { + for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { bittree_reset(coder->match_len_decoder.low[pos_state], LEN_LOW_BITS); bittree_reset(coder->match_len_decoder.mid[pos_state], @@ -949,13 +939,11 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) extern lzma_ret -lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma_lzma_decoder_create(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options) { - const lzma_options_lzma *options = opt; - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); + lz->coder = lzma_alloc(sizeof(lzma_lzma1_decoder), allocator); if (lz->coder == NULL) return LZMA_MEM_ERROR; @@ -966,6 +954,7 @@ lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, // All dictionary sizes are OK here. LZ decoder will take care of // the special cases. + const lzma_options_lzma *options = opt; lz_options->dict_size = options->dict_size; lz_options->preset_dict = options->preset_dict; lz_options->preset_dict_size = options->preset_dict_size; @@ -978,7 +967,7 @@ lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, /// initialization (lzma_lzma_decoder_init() passes function pointer to /// the LZ initialization). static lzma_ret -lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { if (!is_lclppb_valid(options)) @@ -995,7 +984,7 @@ lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { // LZMA can only be the last filter in the chain. This is enforced @@ -1027,7 +1016,8 @@ extern uint64_t lzma_lzma_decoder_memusage_nocheck(const void *options) { const lzma_options_lzma *const opt = options; - return sizeof(lzma_coder) + lzma_lz_decoder_memusage(opt->dict_size); + return sizeof(lzma_lzma1_decoder) + + lzma_lz_decoder_memusage(opt->dict_size); } @@ -1042,15 +1032,14 @@ lzma_lzma_decoder_memusage(const void *options) extern lzma_ret -lzma_lzma_props_decode(void **options, lzma_allocator *allocator, +lzma_lzma_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_lzma *opt; - if (props_size != 5) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_lzma), allocator); + lzma_options_lzma *opt + = lzma_alloc(sizeof(lzma_options_lzma), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h index a463a76..fa8ecb2 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h @@ -19,12 +19,13 @@ /// Allocates and initializes LZMA decoder extern lzma_ret lzma_lzma_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma_decoder_memusage(const void *options); extern lzma_ret lzma_lzma_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); @@ -40,7 +41,7 @@ extern bool lzma_lzma_lclppb_decode( /// Allocate and setup function pointers only. This is used by LZMA1 and /// LZMA2 decoders. extern lzma_ret lzma_lzma_decoder_create( - lzma_lz_decoder *lz, lzma_allocator *allocator, + lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options); /// Gets memory usage without validating lc/lp/pb. This is used by LZMA2 diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c index e8738f4..ba9ce69 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c @@ -28,14 +28,11 @@ literal_matched(lzma_range_encoder *rc, probability *subcoder, symbol += UINT32_C(1) << 8; do { - uint32_t match_bit; - uint32_t subcoder_index; - uint32_t bit; - match_byte <<= 1; - match_bit = match_byte & offset; - subcoder_index = offset + match_bit + (symbol >> 8); - bit = (symbol >> 7) & 1; + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; rc_bit(rc, &subcoder[subcoder_index], bit); symbol <<= 1; @@ -46,7 +43,7 @@ literal_matched(lzma_range_encoder *rc, probability *subcoder, static inline void -literal(lzma_coder *coder, lzma_mf *mf, uint32_t position) +literal(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t position) { // Locate the literal byte to be encoded and the subcoder. const uint8_t cur_byte = mf->buffer[ @@ -80,19 +77,16 @@ literal(lzma_coder *coder, lzma_mf *mf, uint32_t position) static void length_update_prices(lzma_length_encoder *lc, const uint32_t pos_state) { - uint32_t a0, a1, b0, b1; - uint32_t *prices; - uint32_t i; - const uint32_t table_size = lc->table_size; lc->counters[pos_state] = table_size; - a0 = rc_bit_0_price(lc->choice); - a1 = rc_bit_1_price(lc->choice); - b0 = a1 + rc_bit_0_price(lc->choice2); - b1 = a1 + rc_bit_1_price(lc->choice2); - prices = lc->prices[pos_state]; + const uint32_t a0 = rc_bit_0_price(lc->choice); + const uint32_t a1 = rc_bit_1_price(lc->choice); + const uint32_t b0 = a1 + rc_bit_0_price(lc->choice2); + const uint32_t b1 = a1 + rc_bit_1_price(lc->choice2); + uint32_t *const prices = lc->prices[pos_state]; + uint32_t i; for (i = 0; i < table_size && i < LEN_LOW_SYMBOLS; ++i) prices[i] = a0 + rc_bittree_price(lc->low[pos_state], LEN_LOW_BITS, i); @@ -146,39 +140,36 @@ length(lzma_range_encoder *rc, lzma_length_encoder *lc, /////////// static inline void -match(lzma_coder *coder, const uint32_t pos_state, +match(lzma_lzma1_encoder *coder, const uint32_t pos_state, const uint32_t distance, const uint32_t len) { - uint32_t pos_slot; - uint32_t len_to_pos_state; - update_match(coder->state); length(&coder->rc, &coder->match_len_encoder, pos_state, len, coder->fast_mode); - pos_slot = get_pos_slot(distance); - len_to_pos_state = get_len_to_pos_state(len); - rc_bittree(&coder->rc, coder->pos_slot[len_to_pos_state], - POS_SLOT_BITS, pos_slot); + const uint32_t dist_slot = get_dist_slot(distance); + const uint32_t dist_state = get_dist_state(len); + rc_bittree(&coder->rc, coder->dist_slot[dist_state], + DIST_SLOT_BITS, dist_slot); - if (pos_slot >= START_POS_MODEL_INDEX) { - const uint32_t footer_bits = (pos_slot >> 1) - 1; - const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; - const uint32_t pos_reduced = distance - base; + if (dist_slot >= DIST_MODEL_START) { + const uint32_t footer_bits = (dist_slot >> 1) - 1; + const uint32_t base = (2 | (dist_slot & 1)) << footer_bits; + const uint32_t dist_reduced = distance - base; - if (pos_slot < END_POS_MODEL_INDEX) { - // Careful here: base - pos_slot - 1 can be -1, but + if (dist_slot < DIST_MODEL_END) { + // Careful here: base - dist_slot - 1 can be -1, but // rc_bittree_reverse starts at probs[1], not probs[0]. rc_bittree_reverse(&coder->rc, - coder->pos_special + base - pos_slot - 1, - footer_bits, pos_reduced); + coder->dist_special + base - dist_slot - 1, + footer_bits, dist_reduced); } else { - rc_direct(&coder->rc, pos_reduced >> ALIGN_BITS, + rc_direct(&coder->rc, dist_reduced >> ALIGN_BITS, footer_bits - ALIGN_BITS); rc_bittree_reverse( - &coder->rc, coder->pos_align, - ALIGN_BITS, pos_reduced & ALIGN_MASK); + &coder->rc, coder->dist_align, + ALIGN_BITS, dist_reduced & ALIGN_MASK); ++coder->align_price_count; } } @@ -196,7 +187,7 @@ match(lzma_coder *coder, const uint32_t pos_state, //////////////////// static inline void -rep_match(lzma_coder *coder, const uint32_t pos_state, +rep_match(lzma_lzma1_encoder *coder, const uint32_t pos_state, const uint32_t rep, const uint32_t len) { if (rep == 0) { @@ -240,7 +231,7 @@ rep_match(lzma_coder *coder, const uint32_t pos_state, ////////// static void -encode_symbol(lzma_coder *coder, lzma_mf *mf, +encode_symbol(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t back, uint32_t len, uint32_t position) { const uint32_t pos_state = position & coder->pos_mask; @@ -256,7 +247,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); - if (back < REP_DISTANCES) { + if (back < REPS) { // It's a repeated match i.e. the same distance // has been used earlier. rc_bit(&coder->rc, &coder->is_rep[coder->state], 1); @@ -264,7 +255,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, } else { // Normal match rc_bit(&coder->rc, &coder->is_rep[coder->state], 0); - match(coder, pos_state, back - REP_DISTANCES, len); + match(coder, pos_state, back - REPS, len); } } @@ -274,7 +265,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, static bool -encode_init(lzma_coder *coder, lzma_mf *mf) +encode_init(lzma_lzma1_encoder *coder, lzma_mf *mf) { assert(mf_position(mf) == 0); @@ -302,7 +293,7 @@ encode_init(lzma_coder *coder, lzma_mf *mf) static void -encode_eopm(lzma_coder *coder, uint32_t position) +encode_eopm(lzma_lzma1_encoder *coder, uint32_t position) { const uint32_t pos_state = position & coder->pos_mask; rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); @@ -318,23 +309,18 @@ encode_eopm(lzma_coder *coder, uint32_t position) extern lzma_ret -lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma_lzma_encode(lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, uint32_t limit) { - uint32_t position; - // Initialize the stream if no data has been encoded yet. if (!coder->is_initialized && !encode_init(coder, mf)) return LZMA_OK; // Get the lowest bits of the uncompressed offset from the LZ layer. - position = mf_position(mf); + uint32_t position = mf_position(mf); while (true) { - uint32_t len; - uint32_t back; - // Encode pending bits, if any. Calling this before encoding // the next symbol is needed only with plain LZMA, since // LZMA2 always provides big enough buffer to flush @@ -367,12 +353,14 @@ lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Get optimal match (repeat position and length). // Value ranges for pos: - // - [0, REP_DISTANCES): repeated match - // - [REP_DISTANCES, UINT32_MAX): - // match at (pos - REP_DISTANCES) + // - [0, REPS): repeated match + // - [REPS, UINT32_MAX): + // match at (pos - REPS) // - UINT32_MAX: not a match but a literal // Value ranges for len: // - [MATCH_LEN_MIN, MATCH_LEN_MAX] + uint32_t len; + uint32_t back; if (coder->fast_mode) lzma_lzma_optimum_fast(coder, mf, &back, &len); @@ -414,8 +402,8 @@ lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static lzma_ret -lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma_encode(void *coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { // Plain LZMA has no support for sync-flushing. @@ -465,12 +453,10 @@ static void length_encoder_reset(lzma_length_encoder *lencoder, const uint32_t num_pos_states, const bool fast_mode) { - size_t pos_state; - bit_reset(lencoder->choice); bit_reset(lencoder->choice2); - for (pos_state = 0; pos_state < num_pos_states; ++pos_state) { + for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS); bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS); } @@ -478,7 +464,7 @@ length_encoder_reset(lzma_length_encoder *lencoder, bittree_reset(lencoder->high, LEN_HIGH_BITS); if (!fast_mode) - for (pos_state = 0; pos_state < num_pos_states; + for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) length_update_prices(lencoder, pos_state); @@ -487,10 +473,9 @@ length_encoder_reset(lzma_length_encoder *lencoder, extern lzma_ret -lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) +lzma_lzma_encoder_reset(lzma_lzma1_encoder *coder, + const lzma_options_lzma *options) { - size_t i, j; - if (!is_options_valid(options)) return LZMA_OPTIONS_ERROR; @@ -503,14 +488,14 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) // State coder->state = STATE_LIT_LIT; - for (i = 0; i < REP_DISTANCES; ++i) + for (size_t i = 0; i < REPS; ++i) coder->reps[i] = 0; literal_init(coder->literal, options->lc, options->lp); // Bit encoders - for (i = 0; i < STATES; ++i) { - for (j = 0; j <= coder->pos_mask; ++j) { + for (size_t i = 0; i < STATES; ++i) { + for (size_t j = 0; j <= coder->pos_mask; ++j) { bit_reset(coder->is_match[i][j]); bit_reset(coder->is_rep0_long[i][j]); } @@ -521,14 +506,14 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) bit_reset(coder->is_rep2[i]); } - for (i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) - bit_reset(coder->pos_special[i]); + for (size_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i) + bit_reset(coder->dist_special[i]); // Bit tree encoders - for (i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); + for (size_t i = 0; i < DIST_STATES; ++i) + bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS); - bittree_reset(coder->pos_align, ALIGN_BITS); + bittree_reset(coder->dist_align, ALIGN_BITS); // Length encoders length_encoder_reset(&coder->match_len_encoder, @@ -561,20 +546,18 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) extern lzma_ret -lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, +lzma_lzma_encoder_create(void **coder_ptr, + const lzma_allocator *allocator, const lzma_options_lzma *options, lzma_lz_options *lz_options) { - lzma_coder *coder; - uint32_t log_size = 0; - - // Allocate lzma_coder if it wasn't already allocated. + // Allocate lzma_lzma1_encoder if it wasn't already allocated. if (*coder_ptr == NULL) { - *coder_ptr = lzma_alloc(sizeof(lzma_coder), allocator); + *coder_ptr = lzma_alloc(sizeof(lzma_lzma1_encoder), allocator); if (*coder_ptr == NULL) return LZMA_MEM_ERROR; } - coder = *coder_ptr; + lzma_lzma1_encoder *coder = *coder_ptr; // Set compression mode. We haven't validates the options yet, // but it's OK here, since nothing bad happens with invalid @@ -590,6 +573,7 @@ lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, // Set dist_table_size. // Round the dictionary size up to next 2^n. + uint32_t log_size = 0; while ((UINT32_C(1) << log_size) < options->dict_size) ++log_size; @@ -622,7 +606,7 @@ lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, static lzma_ret -lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, +lzma_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { lz->code = &lzma_encode; @@ -632,7 +616,7 @@ lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { return lzma_lz_encoder_init( @@ -643,19 +627,17 @@ lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern uint64_t lzma_lzma_encoder_memusage(const void *options) { - lzma_lz_options lz_options; - uint64_t lz_memusage; - if (!is_options_valid(options)) return UINT64_MAX; + lzma_lz_options lz_options; set_lz_options(&lz_options, options); - lz_memusage = lzma_lz_encoder_memusage(&lz_options); + const uint64_t lz_memusage = lzma_lz_encoder_memusage(&lz_options); if (lz_memusage == UINT64_MAX) return UINT64_MAX; - return (uint64_t)(sizeof(lzma_coder)) + lz_memusage; + return (uint64_t)(sizeof(lzma_lzma1_encoder)) + lz_memusage; } diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h index abb8d8b..6cfdf22 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h @@ -17,8 +17,12 @@ #include "common.h" +typedef struct lzma_lzma1_encoder_s lzma_lzma1_encoder; + + extern lzma_ret lzma_lzma_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma_encoder_memusage(const void *options); @@ -35,18 +39,18 @@ extern bool lzma_lzma_lclppb_encode( /// Initializes raw LZMA encoder; this is used by LZMA2. extern lzma_ret lzma_lzma_encoder_create( - lzma_coder **coder_ptr, lzma_allocator *allocator, + void **coder_ptr, const lzma_allocator *allocator, const lzma_options_lzma *options, lzma_lz_options *lz_options); /// Resets an already initialized LZMA encoder; this is used by LZMA2. extern lzma_ret lzma_lzma_encoder_reset( - lzma_coder *coder, const lzma_options_lzma *options); + lzma_lzma1_encoder *coder, const lzma_options_lzma *options); -extern lzma_ret lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +extern lzma_ret lzma_lzma_encode(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, uint32_t read_limit); #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c index f983126..6c53d2b 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c @@ -10,6 +10,7 @@ /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder_private.h" +#include "memcmplen.h" #define change_pair(small_dist, big_dist) \ @@ -17,17 +18,10 @@ extern void -lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res) +lzma_lzma_optimum_fast(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res) { - const uint8_t *buf; - uint32_t buf_avail; - uint32_t i; - uint32_t rep_len = 0; - uint32_t rep_index = 0; - uint32_t back_main = 0; - uint32_t limit; - const uint32_t nice_len = mf->nice_len; uint32_t len_main; @@ -40,8 +34,8 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m matches_count = coder->matches_count; } - buf = mf_ptr(mf) - 1; - buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); + const uint8_t *buf = mf_ptr(mf) - 1; + const uint32_t buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); if (buf_avail < 2) { // There's not enough input left to encode a match. @@ -51,9 +45,10 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m } // Look for repeated matches; scan the previous four match distances - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len; + uint32_t rep_len = 0; + uint32_t rep_index = 0; + for (uint32_t i = 0; i < REPS; ++i) { // Pointer to the beginning of the match candidate const uint8_t *const buf_back = buf - coder->reps[i] - 1; @@ -64,8 +59,8 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // The first two bytes matched. // Calculate the length of the match. - for (len = 2; len < buf_avail - && buf[len] == buf_back[len]; ++len) ; + const uint32_t len = lzma_memcmplen( + buf, buf_back, 2, buf_avail); // If we have found a repeated match that is at least // nice_len long, return it immediately. @@ -85,13 +80,13 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // We didn't find a long enough repeated match. Encode it as a normal // match if the match length is at least nice_len. if (len_main >= nice_len) { - *back_res = coder->matches[matches_count - 1].dist - + REP_DISTANCES; + *back_res = coder->matches[matches_count - 1].dist + REPS; *len_res = len_main; mf_skip(mf, len_main - 1); return; } + uint32_t back_main = 0; if (len_main >= 2) { back_main = coder->matches[matches_count - 1].dist; @@ -158,27 +153,17 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // the old buf pointer instead of recalculating it with mf_ptr(). ++buf; - limit = len_main - 1; - - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len; - - const uint8_t *const buf_back = buf - coder->reps[i] - 1; - - if (not_equal_16(buf, buf_back)) - continue; - - for (len = 2; len < limit - && buf[len] == buf_back[len]; ++len) ; + const uint32_t limit = my_max(2, len_main - 1); - if (len >= limit) { + for (uint32_t i = 0; i < REPS; ++i) { + if (memcmp(buf, buf - coder->reps[i] - 1, limit) == 0) { *back_res = UINT32_MAX; *len_res = 1; return; } } - *back_res = back_main + REP_DISTANCES; + *back_res = back_main + REPS; *len_res = len_main; mf_skip(mf, len_main - 2); return; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c index d3a6348..59f7734 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c @@ -11,6 +11,7 @@ #include "lzma_encoder_private.h" #include "fastpos.h" +#include "memcmplen.h" //////////// @@ -18,7 +19,7 @@ //////////// static uint32_t -get_literal_price(const lzma_coder *const coder, const uint32_t pos, +get_literal_price(const lzma_lzma1_encoder *const coder, const uint32_t pos, const uint32_t prev_byte, const bool match_mode, uint32_t match_byte, uint32_t symbol) { @@ -35,15 +36,12 @@ get_literal_price(const lzma_coder *const coder, const uint32_t pos, symbol += UINT32_C(1) << 8; do { - uint32_t match_bit; - uint32_t subcoder_index; - uint32_t bit; - match_byte <<= 1; - match_bit = match_byte & offset; - subcoder_index = offset + match_bit + (symbol >> 8); - bit = (symbol >> 7) & 1; + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; price += rc_bit_price(subcoder[subcoder_index], bit); symbol <<= 1; @@ -67,7 +65,7 @@ get_len_price(const lzma_length_encoder *const lencoder, static inline uint32_t -get_short_rep_price(const lzma_coder *const coder, +get_short_rep_price(const lzma_lzma1_encoder *const coder, const lzma_lzma_state state, const uint32_t pos_state) { return rc_bit_0_price(coder->is_rep0[state]) @@ -76,7 +74,7 @@ get_short_rep_price(const lzma_coder *const coder, static inline uint32_t -get_pure_rep_price(const lzma_coder *const coder, const uint32_t rep_index, +get_pure_rep_price(const lzma_lzma1_encoder *const coder, const uint32_t rep_index, const lzma_lzma_state state, uint32_t pos_state) { uint32_t price; @@ -101,7 +99,7 @@ get_pure_rep_price(const lzma_coder *const coder, const uint32_t rep_index, static inline uint32_t -get_rep_price(const lzma_coder *const coder, const uint32_t rep_index, +get_rep_price(const lzma_lzma1_encoder *const coder, const uint32_t rep_index, const uint32_t len, const lzma_lzma_state state, const uint32_t pos_state) { @@ -111,18 +109,18 @@ get_rep_price(const lzma_coder *const coder, const uint32_t rep_index, static inline uint32_t -get_pos_len_price(const lzma_coder *const coder, const uint32_t pos, +get_dist_len_price(const lzma_lzma1_encoder *const coder, const uint32_t dist, const uint32_t len, const uint32_t pos_state) { - const uint32_t len_to_pos_state = get_len_to_pos_state(len); + const uint32_t dist_state = get_dist_state(len); uint32_t price; - if (pos < FULL_DISTANCES) { - price = coder->distances_prices[len_to_pos_state][pos]; + if (dist < FULL_DISTANCES) { + price = coder->dist_prices[dist_state][dist]; } else { - const uint32_t pos_slot = get_pos_slot_2(pos); - price = coder->pos_slot_prices[len_to_pos_state][pos_slot] - + coder->align_prices[pos & ALIGN_MASK]; + const uint32_t dist_slot = get_dist_slot_2(dist); + price = coder->dist_slot_prices[dist_state][dist_slot] + + coder->align_prices[dist & ALIGN_MASK]; } price += get_len_price(&coder->match_len_encoder, len, pos_state); @@ -132,59 +130,53 @@ get_pos_len_price(const lzma_coder *const coder, const uint32_t pos, static void -fill_distances_prices(lzma_coder *coder) +fill_dist_prices(lzma_lzma1_encoder *coder) { - uint32_t len_to_pos_state; - uint32_t pos_slot; - uint32_t i; - - for (len_to_pos_state = 0; - len_to_pos_state < LEN_TO_POS_STATES; - ++len_to_pos_state) { + for (uint32_t dist_state = 0; dist_state < DIST_STATES; ++dist_state) { - uint32_t *const pos_slot_prices - = coder->pos_slot_prices[len_to_pos_state]; + uint32_t *const dist_slot_prices + = coder->dist_slot_prices[dist_state]; - // Price to encode the pos_slot. - for (pos_slot = 0; - pos_slot < coder->dist_table_size; ++pos_slot) - pos_slot_prices[pos_slot] = rc_bittree_price( - coder->pos_slot[len_to_pos_state], - POS_SLOT_BITS, pos_slot); + // Price to encode the dist_slot. + for (uint32_t dist_slot = 0; + dist_slot < coder->dist_table_size; ++dist_slot) + dist_slot_prices[dist_slot] = rc_bittree_price( + coder->dist_slot[dist_state], + DIST_SLOT_BITS, dist_slot); // For matches with distance >= FULL_DISTANCES, add the price // of the direct bits part of the match distance. (Align bits // are handled by fill_align_prices()). - for (pos_slot = END_POS_MODEL_INDEX; - pos_slot < coder->dist_table_size; ++pos_slot) - pos_slot_prices[pos_slot] += rc_direct_price( - ((pos_slot >> 1) - 1) - ALIGN_BITS); + for (uint32_t dist_slot = DIST_MODEL_END; + dist_slot < coder->dist_table_size; + ++dist_slot) + dist_slot_prices[dist_slot] += rc_direct_price( + ((dist_slot >> 1) - 1) - ALIGN_BITS); // Distances in the range [0, 3] are fully encoded with - // pos_slot, so they are used for coder->distances_prices + // dist_slot, so they are used for coder->dist_prices // as is. - for (i = 0; i < START_POS_MODEL_INDEX; ++i) - coder->distances_prices[len_to_pos_state][i] - = pos_slot_prices[i]; + for (uint32_t i = 0; i < DIST_MODEL_START; ++i) + coder->dist_prices[dist_state][i] + = dist_slot_prices[i]; } - // Distances in the range [4, 127] depend on pos_slot and pos_special. - // We do this in a loop separate from the above loop to avoid - // redundant calls to get_pos_slot(). - for (i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) { - const uint32_t pos_slot = get_pos_slot(i); - const uint32_t footer_bits = ((pos_slot >> 1) - 1); - const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; + // Distances in the range [4, 127] depend on dist_slot and + // dist_special. We do this in a loop separate from the above + // loop to avoid redundant calls to get_dist_slot(). + for (uint32_t i = DIST_MODEL_START; i < FULL_DISTANCES; ++i) { + const uint32_t dist_slot = get_dist_slot(i); + const uint32_t footer_bits = ((dist_slot >> 1) - 1); + const uint32_t base = (2 | (dist_slot & 1)) << footer_bits; const uint32_t price = rc_bittree_reverse_price( - coder->pos_special + base - pos_slot - 1, + coder->dist_special + base - dist_slot - 1, footer_bits, i - base); - for (len_to_pos_state = 0; - len_to_pos_state < LEN_TO_POS_STATES; - ++len_to_pos_state) - coder->distances_prices[len_to_pos_state][i] - = price + coder->pos_slot_prices[ - len_to_pos_state][pos_slot]; + for (uint32_t dist_state = 0; dist_state < DIST_STATES; + ++dist_state) + coder->dist_prices[dist_state][i] + = price + coder->dist_slot_prices[ + dist_state][dist_slot]; } coder->match_price_count = 0; @@ -193,12 +185,11 @@ fill_distances_prices(lzma_coder *coder) static void -fill_align_prices(lzma_coder *coder) +fill_align_prices(lzma_lzma1_encoder *coder) { - uint32_t i; - for (i = 0; i < ALIGN_TABLE_SIZE; ++i) + for (uint32_t i = 0; i < ALIGN_SIZE; ++i) coder->align_prices[i] = rc_bittree_reverse_price( - coder->pos_align, ALIGN_BITS, i); + coder->dist_align, ALIGN_BITS, i); coder->align_price_count = 0; return; @@ -230,18 +221,15 @@ make_short_rep(lzma_optimal *optimal) static void -backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, - uint32_t *LZMA_RESTRICT back_res, uint32_t cur) +backward(lzma_lzma1_encoder *restrict coder, uint32_t *restrict len_res, + uint32_t *restrict back_res, uint32_t cur) { + coder->opts_end_index = cur; + uint32_t pos_mem = coder->opts[cur].pos_prev; uint32_t back_mem = coder->opts[cur].back_prev; - coder->opts_end_index = cur; - do { - const uint32_t pos_prev = pos_mem; - const uint32_t back_cur = back_mem; - if (coder->opts[cur].prev_1_is_literal) { make_literal(&coder->opts[pos_mem]); coder->opts[pos_mem].pos_prev = pos_mem - 1; @@ -256,6 +244,9 @@ backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, } } + const uint32_t pos_prev = pos_mem; + const uint32_t back_cur = back_mem; + back_mem = coder->opts[pos_prev].back_prev; pos_mem = coder->opts[pos_prev].pos_prev; @@ -278,27 +269,10 @@ backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, ////////// static inline uint32_t -helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res, +helper1(lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, uint32_t position) { - uint32_t buf_avail; - const uint8_t *buf; - uint32_t rep_lens[REP_DISTANCES]; - uint32_t rep_max_index = 0; - uint32_t i; - - uint8_t current_byte; - uint8_t match_byte; - - uint32_t pos_state; - uint32_t match_price; - uint32_t rep_match_price; - uint32_t len_end; - uint32_t len; - - uint32_t normal_match_price; - const uint32_t nice_len = mf->nice_len; uint32_t len_main; @@ -312,18 +286,19 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, matches_count = coder->matches_count; } - buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); + const uint32_t buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); if (buf_avail < 2) { *back_res = UINT32_MAX; *len_res = 1; return UINT32_MAX; } - buf = mf_ptr(mf) - 1; + const uint8_t *const buf = mf_ptr(mf) - 1; - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len_test; + uint32_t rep_lens[REPS]; + uint32_t rep_max_index = 0; + for (uint32_t i = 0; i < REPS; ++i) { const uint8_t *const buf_back = buf - coder->reps[i] - 1; if (not_equal_16(buf, buf_back)) { @@ -331,12 +306,9 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, continue; } - for (len_test = 2; len_test < buf_avail - && buf[len_test] == buf_back[len_test]; - ++len_test) ; + rep_lens[i] = lzma_memcmplen(buf, buf_back, 2, buf_avail); - rep_lens[i] = len_test; - if (len_test > rep_lens[rep_max_index]) + if (rep_lens[i] > rep_lens[rep_max_index]) rep_max_index = i; } @@ -349,15 +321,14 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, if (len_main >= nice_len) { - *back_res = coder->matches[matches_count - 1].dist - + REP_DISTANCES; + *back_res = coder->matches[matches_count - 1].dist + REPS; *len_res = len_main; mf_skip(mf, len_main - 1); return UINT32_MAX; } - current_byte = *buf; - match_byte = *(buf - coder->reps[0] - 1); + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - coder->reps[0] - 1); if (len_main < 2 && current_byte != match_byte && rep_lens[rep_max_index] < 2) { @@ -368,7 +339,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, coder->opts[0].state = coder->state; - pos_state = position & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; coder->opts[1].price = rc_bit_0_price( coder->is_match[coder->state][pos_state]) @@ -378,9 +349,9 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, make_literal(&coder->opts[1]); - match_price = rc_bit_1_price( + const uint32_t match_price = rc_bit_1_price( coder->is_match[coder->state][pos_state]); - rep_match_price = match_price + const uint32_t rep_match_price = match_price + rc_bit_1_price(coder->is_rep[coder->state]); if (match_byte == current_byte) { @@ -394,7 +365,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, } } - len_end = my_max(len_main, rep_lens[rep_max_index]); + const uint32_t len_end = my_max(len_main, rep_lens[rep_max_index]); if (len_end < 2) { *back_res = coder->opts[1].back_prev; @@ -404,23 +375,21 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, coder->opts[1].pos_prev = 0; - for (i = 0; i < REP_DISTANCES; ++i) + for (uint32_t i = 0; i < REPS; ++i) coder->opts[0].backs[i] = coder->reps[i]; - len = len_end; + uint32_t len = len_end; do { coder->opts[len].price = RC_INFINITY_PRICE; } while (--len >= 2); - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t price; - + for (uint32_t i = 0; i < REPS; ++i) { uint32_t rep_len = rep_lens[i]; if (rep_len < 2) continue; - price = rep_match_price + get_pure_rep_price( + const uint32_t price = rep_match_price + get_pure_rep_price( coder, i, coder->state, pos_state); do { @@ -439,7 +408,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, } - normal_match_price = match_price + const uint32_t normal_match_price = match_price + rc_bit_0_price(coder->is_rep[coder->state]); len = rep_lens[0] >= 2 ? rep_lens[0] + 1 : 2; @@ -451,14 +420,13 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, for(; ; ++len) { const uint32_t dist = coder->matches[i].dist; const uint32_t cur_and_len_price = normal_match_price - + get_pos_len_price(coder, + + get_dist_len_price(coder, dist, len, pos_state); if (cur_and_len_price < coder->opts[len].price) { coder->opts[len].price = cur_and_len_price; coder->opts[len].pos_prev = 0; - coder->opts[len].back_prev - = dist + REP_DISTANCES; + coder->opts[len].back_prev = dist + REPS; coder->opts[len].prev_1_is_literal = false; } @@ -473,7 +441,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static inline uint32_t -helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, +helper2(lzma_lzma1_encoder *coder, uint32_t *reps, const uint8_t *buf, uint32_t len_end, uint32_t position, const uint32_t cur, const uint32_t nice_len, const uint32_t buf_avail_full) { @@ -481,19 +449,6 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, uint32_t new_len = coder->longest_match_length; uint32_t pos_prev = coder->opts[cur].pos_prev; lzma_lzma_state state; - uint32_t buf_avail; - uint32_t rep_index; - uint32_t i; - - uint32_t cur_price; - uint8_t current_byte; - uint8_t match_byte; - uint32_t pos_state; - uint32_t cur_and_1_price; - bool next_is_literal = false; - uint32_t match_price; - uint32_t rep_match_price; - uint32_t start_len = 2; if (coder->opts[cur].prev_1_is_literal) { --pos_prev; @@ -501,7 +456,7 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (coder->opts[cur].prev_2) { state = coder->opts[coder->opts[cur].pos_prev_2].state; - if (coder->opts[cur].back_prev_2 < REP_DISTANCES) + if (coder->opts[cur].back_prev_2 < REPS) update_long_rep(state); else update_match(state); @@ -530,48 +485,49 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, update_long_rep(state); } else { pos = coder->opts[cur].back_prev; - if (pos < REP_DISTANCES) + if (pos < REPS) update_long_rep(state); else update_match(state); } - if (pos < REP_DISTANCES) { - uint32_t i; - + if (pos < REPS) { reps[0] = coder->opts[pos_prev].backs[pos]; + uint32_t i; for (i = 1; i <= pos; ++i) reps[i] = coder->opts[pos_prev].backs[i - 1]; - for (; i < REP_DISTANCES; ++i) + for (; i < REPS; ++i) reps[i] = coder->opts[pos_prev].backs[i]; } else { - reps[0] = pos - REP_DISTANCES; + reps[0] = pos - REPS; - for (i = 1; i < REP_DISTANCES; ++i) + for (uint32_t i = 1; i < REPS; ++i) reps[i] = coder->opts[pos_prev].backs[i - 1]; } } coder->opts[cur].state = state; - for (i = 0; i < REP_DISTANCES; ++i) + for (uint32_t i = 0; i < REPS; ++i) coder->opts[cur].backs[i] = reps[i]; - cur_price = coder->opts[cur].price; + const uint32_t cur_price = coder->opts[cur].price; - current_byte = *buf; - match_byte = *(buf - reps[0] - 1); + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - reps[0] - 1); - pos_state = position & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; - cur_and_1_price = cur_price + const uint32_t cur_and_1_price = cur_price + rc_bit_0_price(coder->is_match[state][pos_state]) + get_literal_price(coder, position, buf[-1], !is_literal_state(state), match_byte, current_byte); + bool next_is_literal = false; + if (cur_and_1_price < coder->opts[cur + 1].price) { coder->opts[cur + 1].price = cur_and_1_price; coder->opts[cur + 1].pos_prev = cur; @@ -579,9 +535,9 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, next_is_literal = true; } - match_price = cur_price + const uint32_t match_price = cur_price + rc_bit_1_price(coder->is_match[state][pos_state]); - rep_match_price = match_price + const uint32_t rep_match_price = match_price + rc_bit_1_price(coder->is_rep[state]); if (match_byte == current_byte @@ -602,40 +558,31 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (buf_avail_full < 2) return len_end; - buf_avail = my_min(buf_avail_full, nice_len); + const uint32_t buf_avail = my_min(buf_avail_full, nice_len); if (!next_is_literal && match_byte != current_byte) { // speed optimization // try literal + rep0 const uint8_t *const buf_back = buf - reps[0] - 1; const uint32_t limit = my_min(buf_avail_full, nice_len + 1); - uint32_t len_test = 1; - while (len_test < limit && buf[len_test] == buf_back[len_test]) - ++len_test; - - --len_test; + const uint32_t len_test = lzma_memcmplen(buf, buf_back, 1, limit) - 1; if (len_test >= 2) { - uint32_t pos_state_next; - uint32_t next_rep_match_price; - uint32_t offset; - uint32_t cur_and_len_price; - lzma_lzma_state state_2 = state; update_literal(state_2); - pos_state_next = (position + 1) & coder->pos_mask; - next_rep_match_price = cur_and_1_price + const uint32_t pos_state_next = (position + 1) & coder->pos_mask; + const uint32_t next_rep_match_price = cur_and_1_price + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); //for (; len_test >= 2; --len_test) { - offset = cur + 1 + len_test; + const uint32_t offset = cur + 1 + len_test; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; - cur_and_len_price = next_rep_match_price + const uint32_t cur_and_len_price = next_rep_match_price + get_rep_price(coder, 0, len_test, state_2, pos_state_next); @@ -651,23 +598,20 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, } - for (rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) { - uint32_t len_test, len_test_2, len_test_temp; - uint32_t price, limit; + uint32_t start_len = 2; // speed optimization + for (uint32_t rep_index = 0; rep_index < REPS; ++rep_index) { const uint8_t *const buf_back = buf - reps[rep_index] - 1; if (not_equal_16(buf, buf_back)) continue; - for (len_test = 2; len_test < buf_avail - && buf[len_test] == buf_back[len_test]; - ++len_test) ; + uint32_t len_test = lzma_memcmplen(buf, buf_back, 2, buf_avail); while (len_end < cur + len_test) coder->opts[++len_end].price = RC_INFINITY_PRICE; - len_test_temp = len_test; - price = rep_match_price + get_pure_rep_price( + const uint32_t len_test_temp = len_test; + const uint32_t price = rep_match_price + get_pure_rep_price( coder, rep_index, state, pos_state); do { @@ -689,8 +633,8 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, start_len = len_test + 1; - len_test_2 = len_test + 1; - limit = my_min(buf_avail_full, + uint32_t len_test_2 = len_test + 1; + const uint32_t limit = my_min(buf_avail_full, len_test_2 + nice_len); for (; len_test_2 < limit && buf[len_test_2] == buf_back[len_test_2]; @@ -699,18 +643,12 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, len_test_2 -= len_test + 1; if (len_test_2 >= 2) { - uint32_t pos_state_next; - uint32_t cur_and_len_literal_price; - uint32_t next_rep_match_price; - uint32_t offset; - uint32_t cur_and_len_price; - lzma_lzma_state state_2 = state; update_long_rep(state_2); - pos_state_next = (position + len_test) & coder->pos_mask; + uint32_t pos_state_next = (position + len_test) & coder->pos_mask; - cur_and_len_literal_price = price + const uint32_t cur_and_len_literal_price = price + get_len_price(&coder->rep_len_encoder, len_test, pos_state) + rc_bit_0_price(coder->is_match[state_2][pos_state_next]) @@ -722,17 +660,17 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, pos_state_next = (position + len_test + 1) & coder->pos_mask; - next_rep_match_price = cur_and_len_literal_price + const uint32_t next_rep_match_price = cur_and_len_literal_price + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); //for(; len_test_2 >= 2; len_test_2--) { - offset = cur + len_test + 1 + len_test_2; + const uint32_t offset = cur + len_test + 1 + len_test_2; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; - cur_and_len_price = next_rep_match_price + const uint32_t cur_and_len_price = next_rep_match_price + get_rep_price(coder, 0, len_test_2, state_2, pos_state_next); @@ -763,29 +701,27 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (new_len >= start_len) { - uint32_t len_test; - uint32_t i = 0; - const uint32_t normal_match_price = match_price + rc_bit_0_price(coder->is_rep[state]); while (len_end < cur + new_len) coder->opts[++len_end].price = RC_INFINITY_PRICE; + uint32_t i = 0; while (start_len > coder->matches[i].len) ++i; - for (len_test = start_len; ; ++len_test) { + for (uint32_t len_test = start_len; ; ++len_test) { const uint32_t cur_back = coder->matches[i].dist; uint32_t cur_and_len_price = normal_match_price - + get_pos_len_price(coder, + + get_dist_len_price(coder, cur_back, len_test, pos_state); if (cur_and_len_price < coder->opts[cur + len_test].price) { coder->opts[cur + len_test].price = cur_and_len_price; coder->opts[cur + len_test].pos_prev = cur; coder->opts[cur + len_test].back_prev - = cur_back + REP_DISTANCES; + = cur_back + REPS; coder->opts[cur + len_test].prev_1_is_literal = false; } @@ -803,16 +739,12 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, len_test_2 -= len_test + 1; if (len_test_2 >= 2) { - uint32_t pos_state_next; - uint32_t cur_and_len_literal_price; - uint32_t next_rep_match_price; - uint32_t offset; - lzma_lzma_state state_2 = state; update_match(state_2); - pos_state_next = (position + len_test) & coder->pos_mask; + uint32_t pos_state_next + = (position + len_test) & coder->pos_mask; - cur_and_len_literal_price = cur_and_len_price + const uint32_t cur_and_len_literal_price = cur_and_len_price + rc_bit_0_price( coder->is_match[state_2][pos_state_next]) + get_literal_price(coder, @@ -825,14 +757,14 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, update_literal(state_2); pos_state_next = (pos_state_next + 1) & coder->pos_mask; - next_rep_match_price + const uint32_t next_rep_match_price = cur_and_len_literal_price + rc_bit_1_price( coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); // for(; len_test_2 >= 2; --len_test_2) { - offset = cur + len_test + 1 + len_test_2; + const uint32_t offset = cur + len_test + 1 + len_test_2; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; @@ -849,7 +781,7 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, coder->opts[offset].prev_2 = true; coder->opts[offset].pos_prev_2 = cur; coder->opts[offset].back_prev_2 - = cur_back + REP_DISTANCES; + = cur_back + REPS; } //} } @@ -865,14 +797,11 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, extern void -lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res, +lzma_lzma_optimum_normal(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, uint32_t position) { - uint32_t reps[REP_DISTANCES]; - uint32_t len_end; - uint32_t cur; - // If we have symbols pending, return the next pending symbol. if (coder->opts_end_index != coder->opts_current_index) { assert(mf->read_ahead > 0); @@ -889,9 +818,9 @@ lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT // In liblzma they were moved into this single place. if (mf->read_ahead == 0) { if (coder->match_price_count >= (1 << 7)) - fill_distances_prices(coder); + fill_dist_prices(coder); - if (coder->align_price_count >= ALIGN_TABLE_SIZE) + if (coder->align_price_count >= ALIGN_SIZE) fill_align_prices(coder); } @@ -899,13 +828,14 @@ lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT // the original function into two pieces makes it at least a little // more readable, since those two parts don't share many variables. - len_end = helper1(coder, mf, back_res, len_res, position); + uint32_t len_end = helper1(coder, mf, back_res, len_res, position); if (len_end == UINT32_MAX) return; - + uint32_t reps[REPS]; memcpy(reps, coder->reps, sizeof(reps)); + uint32_t cur; for (cur = 1; cur < len_end; ++cur) { assert(cur < OPTS); diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c index 9332abf..711df02 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c @@ -2,6 +2,7 @@ // /// \file lzma_encoder_presets.c /// \brief Encoder presets +/// \note xz needs this even when only decoding is enabled. // // Author: Lasse Collin // @@ -16,10 +17,6 @@ extern LZMA_API(lzma_bool) lzma_lzma_preset(lzma_options_lzma *options, uint32_t preset) { - static const uint8_t dict_pow2[] - = { 18, 20, 21, 22, 22, 23, 23, 24, 25, 26 }; - static const uint8_t depths[] = { 4, 8, 24, 48 }; - const uint32_t level = preset & LZMA_PRESET_LEVEL_MASK; const uint32_t flags = preset & ~LZMA_PRESET_LEVEL_MASK; const uint32_t supported_flags = LZMA_PRESET_EXTREME; @@ -34,12 +31,15 @@ lzma_lzma_preset(lzma_options_lzma *options, uint32_t preset) options->lp = LZMA_LP_DEFAULT; options->pb = LZMA_PB_DEFAULT; + static const uint8_t dict_pow2[] + = { 18, 20, 21, 22, 22, 23, 23, 24, 25, 26 }; options->dict_size = UINT32_C(1) << dict_pow2[level]; if (level <= 3) { options->mode = LZMA_MODE_FAST; options->mf = level == 0 ? LZMA_MF_HC3 : LZMA_MF_HC4; options->nice_len = level <= 1 ? 128 : 273; + static const uint8_t depths[] = { 4, 8, 24, 48 }; options->depth = depths[level]; } else { options->mode = LZMA_MODE_NORMAL; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h index 04fb29e..a2da969 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h @@ -64,12 +64,12 @@ typedef struct { uint32_t pos_prev; // pos_next; uint32_t back_prev; - uint32_t backs[REP_DISTANCES]; + uint32_t backs[REPS]; } lzma_optimal; -struct lzma_coder_s { +struct lzma_lzma1_encoder_s { /// Range encoder lzma_range_encoder rc; @@ -77,7 +77,7 @@ struct lzma_coder_s { lzma_lzma_state state; /// The four most recent match distances - uint32_t reps[REP_DISTANCES]; + uint32_t reps[REPS]; /// Array of match candidates lzma_match matches[MATCH_LEN_MAX + 1]; @@ -112,9 +112,9 @@ struct lzma_coder_s { probability is_rep1[STATES]; probability is_rep2[STATES]; probability is_rep0_long[STATES][POS_STATES_MAX]; - probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; - probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; - probability pos_align[ALIGN_TABLE_SIZE]; + probability dist_slot[DIST_STATES][DIST_SLOTS]; + probability dist_special[FULL_DISTANCES - DIST_MODEL_END]; + probability dist_align[ALIGN_SIZE]; // These are the same as in lzma_decoder.c except that the encoders // include also price tables. @@ -122,12 +122,12 @@ struct lzma_coder_s { lzma_length_encoder rep_len_encoder; // Price tables - uint32_t pos_slot_prices[LEN_TO_POS_STATES][POS_SLOTS]; - uint32_t distances_prices[LEN_TO_POS_STATES][FULL_DISTANCES]; + uint32_t dist_slot_prices[DIST_STATES][DIST_SLOTS]; + uint32_t dist_prices[DIST_STATES][FULL_DISTANCES]; uint32_t dist_table_size; uint32_t match_price_count; - uint32_t align_prices[ALIGN_TABLE_SIZE]; + uint32_t align_prices[ALIGN_SIZE]; uint32_t align_price_count; // Optimal @@ -138,11 +138,11 @@ struct lzma_coder_s { extern void lzma_lzma_optimum_fast( - lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res); + lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res); -extern void lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint32_t *LZMA_RESTRICT back_res, - uint32_t *LZMA_RESTRICT len_res, uint32_t position); +extern void lzma_lzma_optimum_normal(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, uint32_t *restrict back_res, + uint32_t *restrict len_res, uint32_t position); #endif diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_common.h b/Utilities/cmliblzma/liblzma/rangecoder/range_common.h index f15623e..2c74dc1 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_common.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_common.h @@ -40,11 +40,8 @@ // This does the same for a complete bit tree. // (A tree represented as an array.) #define bittree_reset(probs, bit_levels) \ - do { \ - uint32_t bt_i; \ - for (bt_i = 0; bt_i < (1 << (bit_levels)); ++bt_i) \ - bit_reset((probs)[bt_i]); \ - } while (0) + for (uint32_t bt_i = 0; bt_i < (1 << (bit_levels)); ++bt_i) \ + bit_reset((probs)[bt_i]) ////////////////////// diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h b/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h index 199e7b5..e0b051f 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h @@ -25,20 +25,26 @@ typedef struct { /// Reads the first five bytes to initialize the range decoder. -static inline bool -rc_read_init(lzma_range_decoder *rc, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size) +static inline lzma_ret +rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) { while (rc->init_bytes_left > 0) { if (*in_pos == in_size) - return false; + return LZMA_OK; + + // The first byte is always 0x00. It could have been omitted + // in LZMA2 but it wasn't, so one byte is wasted in every + // LZMA2 chunk. + if (rc->init_bytes_left == 5 && in[*in_pos] != 0x00) + return LZMA_DATA_ERROR; rc->code = (rc->code << 8) | in[*in_pos]; ++*in_pos; --rc->init_bytes_left; } - return true; + return LZMA_STREAM_END; } diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h b/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h index e9614f2..1e1c369 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h @@ -115,8 +115,7 @@ rc_direct(lzma_range_encoder *rc, static inline void rc_flush(lzma_range_encoder *rc) { - size_t i; - for (i = 0; i < 5; ++i) + for (size_t i = 0; i < 5; ++i) rc->symbols[rc->count++] = RC_FLUSH; } diff --git a/Utilities/cmliblzma/liblzma/simple/arm.c b/Utilities/cmliblzma/liblzma/simple/arm.c index 8dcba39..181d0e3 100644 --- a/Utilities/cmliblzma/liblzma/simple/arm.c +++ b/Utilities/cmliblzma/liblzma/simple/arm.c @@ -15,19 +15,19 @@ static size_t -arm_code(lzma_simple *simple lzma_attribute((__unused__)), +arm_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { size_t i; for (i = 0; i + 4 <= size; i += 4) { if (buffer[i + 3] == 0xEB) { - uint32_t dest; uint32_t src = (buffer[i + 2] << 16) | (buffer[i + 1] << 8) | (buffer[i + 0]); src <<= 2; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + 8 + src; else @@ -45,7 +45,7 @@ arm_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -arm_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +arm_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -54,7 +54,8 @@ arm_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_arm_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_arm_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return arm_coder_init(next, allocator, filters, true); @@ -62,7 +63,8 @@ lzma_simple_arm_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_arm_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_arm_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return arm_coder_init(next, allocator, filters, false); diff --git a/Utilities/cmliblzma/liblzma/simple/armthumb.c b/Utilities/cmliblzma/liblzma/simple/armthumb.c index 4b890a3..eab4862 100644 --- a/Utilities/cmliblzma/liblzma/simple/armthumb.c +++ b/Utilities/cmliblzma/liblzma/simple/armthumb.c @@ -15,7 +15,7 @@ static size_t -armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), +armthumb_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -23,7 +23,6 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), for (i = 0; i + 4 <= size; i += 2) { if ((buffer[i + 1] & 0xF8) == 0xF0 && (buffer[i + 3] & 0xF8) == 0xF8) { - uint32_t dest; uint32_t src = ((buffer[i + 1] & 0x7) << 19) | (buffer[i + 0] << 11) | ((buffer[i + 3] & 0x7) << 8) @@ -31,6 +30,7 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), src <<= 1; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + 4 + src; else @@ -50,7 +50,7 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -armthumb_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +armthumb_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -60,7 +60,8 @@ armthumb_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_armthumb_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return armthumb_coder_init(next, allocator, filters, true); } @@ -68,7 +69,8 @@ lzma_simple_armthumb_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return armthumb_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/ia64.c b/Utilities/cmliblzma/liblzma/simple/ia64.c index c537cac..580529e 100644 --- a/Utilities/cmliblzma/liblzma/simple/ia64.c +++ b/Utilities/cmliblzma/liblzma/simple/ia64.c @@ -15,7 +15,7 @@ static size_t -ia64_code(lzma_simple *simple lzma_attribute((__unused__)), +ia64_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -28,42 +28,36 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), size_t i; for (i = 0; i + 16 <= size; i += 16) { - size_t slot; - const uint32_t instr_template = buffer[i] & 0x1F; const uint32_t mask = BRANCH_TABLE[instr_template]; uint32_t bit_pos = 5; - for (slot = 0; slot < 3; ++slot, bit_pos += 41) { + for (size_t slot = 0; slot < 3; ++slot, bit_pos += 41) { + if (((mask >> slot) & 1) == 0) + continue; + const size_t byte_pos = (bit_pos >> 3); const uint32_t bit_res = bit_pos & 0x7; uint64_t instruction = 0; - uint64_t inst_norm; - size_t j; - - if (((mask >> slot) & 1) == 0) - continue; - for (j = 0; j < 6; ++j) + for (size_t j = 0; j < 6; ++j) instruction += (uint64_t)( buffer[i + j + byte_pos]) << (8 * j); - inst_norm = instruction >> bit_res; + uint64_t inst_norm = instruction >> bit_res; if (((inst_norm >> 37) & 0xF) == 0x5 && ((inst_norm >> 9) & 0x7) == 0 /* && (inst_norm & 0x3F)== 0 */ ) { - uint32_t dest; - size_t j; - uint32_t src = (uint32_t)( (inst_norm >> 13) & 0xFFFFF); src |= ((inst_norm >> 36) & 1) << 20; src <<= 4; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + src; else @@ -79,7 +73,7 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), instruction &= (1 << bit_res) - 1; instruction |= (inst_norm << bit_res); - for (j = 0; j < 6; j++) + for (size_t j = 0; j < 6; j++) buffer[i + j + byte_pos] = (uint8_t)( instruction >> (8 * j)); @@ -92,7 +86,7 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -ia64_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +ia64_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -102,7 +96,8 @@ ia64_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_ia64_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return ia64_coder_init(next, allocator, filters, true); } @@ -110,7 +105,8 @@ lzma_simple_ia64_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_ia64_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return ia64_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/powerpc.c b/Utilities/cmliblzma/liblzma/simple/powerpc.c index 6f83511..54dfbf1 100644 --- a/Utilities/cmliblzma/liblzma/simple/powerpc.c +++ b/Utilities/cmliblzma/liblzma/simple/powerpc.c @@ -15,7 +15,7 @@ static size_t -powerpc_code(lzma_simple *simple lzma_attribute((__unused__)), +powerpc_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -49,7 +49,7 @@ powerpc_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -powerpc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +powerpc_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -59,7 +59,8 @@ powerpc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_powerpc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return powerpc_coder_init(next, allocator, filters, true); } @@ -67,7 +68,8 @@ lzma_simple_powerpc_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_powerpc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return powerpc_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/simple_coder.c b/Utilities/cmliblzma/liblzma/simple/simple_coder.c index f3bbdd7..13ebabc 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_coder.c +++ b/Utilities/cmliblzma/liblzma/simple/simple_coder.c @@ -18,10 +18,10 @@ /// Copied or encodes/decodes more data to out[]. static lzma_ret -copy_or_code(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +copy_or_code(lzma_simple_coder *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { assert(!coder->end_was_reached); @@ -55,7 +55,7 @@ copy_or_code(lzma_coder *coder, lzma_allocator *allocator, static size_t -call_filter(lzma_coder *coder, uint8_t *buffer, size_t size) +call_filter(lzma_simple_coder *coder, uint8_t *buffer, size_t size) { const size_t filtered = coder->filter(coder->simple, coder->now_pos, coder->is_encoder, @@ -66,13 +66,12 @@ call_filter(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -simple_code(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +simple_code(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { - size_t out_avail; - size_t buf_avail; + lzma_simple_coder *coder = coder_ptr; // TODO: Add partial support for LZMA_SYNC_FLUSH. We can support it // in cases when the filter is able to filter everything. With most @@ -108,13 +107,9 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, // more data to out[] hopefully filling it completely. Then filter // the data in out[]. This step is where most of the data gets // filtered if the buffer sizes used by the application are reasonable. - out_avail = out_size - *out_pos; - buf_avail = coder->size - coder->pos; + const size_t out_avail = out_size - *out_pos; + const size_t buf_avail = coder->size - coder->pos; if (out_avail > buf_avail || buf_avail == 0) { - size_t size; - size_t filtered; - size_t unfiltered; - // Store the old position so that we know from which byte // to start filtering. const size_t out_start = *out_pos; @@ -137,10 +132,11 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, } // Filter out[]. - size = *out_pos - out_start; - filtered = call_filter(coder, out + out_start, size); + const size_t size = *out_pos - out_start; + const size_t filtered = call_filter( + coder, out + out_start, size); - unfiltered = size - filtered; + const size_t unfiltered = size - filtered; assert(unfiltered <= coder->allocated / 2); // Now we can update coder->pos and coder->size, because @@ -204,8 +200,9 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, static void -simple_coder_end(lzma_coder *coder, lzma_allocator *allocator) +simple_coder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_simple_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder->simple, allocator); lzma_free(coder, allocator); @@ -214,10 +211,12 @@ simple_coder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -simple_coder_update(lzma_coder *coder, lzma_allocator *allocator, +simple_coder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_simple_coder *coder = coder_ptr; + // No update support, just call the next filter in the chain. return lzma_next_filter_update( &coder->next, allocator, reversed_filters + 1); @@ -225,59 +224,59 @@ simple_coder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_simple_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size), size_t simple_size, size_t unfiltered_max, uint32_t alignment, bool is_encoder) { - // Allocate memory for the lzma_coder structure if needed. - if (next->coder == NULL) { + // Allocate memory for the lzma_simple_coder structure if needed. + lzma_simple_coder *coder = next->coder; + if (coder == NULL) { // Here we allocate space also for the temporary buffer. We // need twice the size of unfiltered_max, because then it // is always possible to filter at least unfiltered_max bytes // more data in coder->buffer[] if it can be filled completely. - next->coder = lzma_alloc(sizeof(lzma_coder) + coder = lzma_alloc(sizeof(lzma_simple_coder) + 2 * unfiltered_max, allocator); - if (next->coder == NULL) + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &simple_code; next->end = &simple_coder_end; next->update = &simple_coder_update; - next->coder->next = LZMA_NEXT_CODER_INIT; - next->coder->filter = filter; - next->coder->allocated = 2 * unfiltered_max; + coder->next = LZMA_NEXT_CODER_INIT; + coder->filter = filter; + coder->allocated = 2 * unfiltered_max; // Allocate memory for filter-specific data structure. if (simple_size > 0) { - next->coder->simple = lzma_alloc( - simple_size, allocator); - if (next->coder->simple == NULL) + coder->simple = lzma_alloc(simple_size, allocator); + if (coder->simple == NULL) return LZMA_MEM_ERROR; } else { - next->coder->simple = NULL; + coder->simple = NULL; } } if (filters[0].options != NULL) { const lzma_options_bcj *simple = filters[0].options; - next->coder->now_pos = simple->start_offset; - if (next->coder->now_pos & (alignment - 1)) + coder->now_pos = simple->start_offset; + if (coder->now_pos & (alignment - 1)) return LZMA_OPTIONS_ERROR; } else { - next->coder->now_pos = 0; + coder->now_pos = 0; } // Reset variables. - next->coder->is_encoder = is_encoder; - next->coder->end_was_reached = false; - next->coder->pos = 0; - next->coder->filtered = 0; - next->coder->size = 0; - - return lzma_next_filter_init( - &next->coder->next, allocator, filters + 1); + coder->is_encoder = is_encoder; + coder->end_was_reached = false; + coder->pos = 0; + coder->filtered = 0; + coder->size = 0; + + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } diff --git a/Utilities/cmliblzma/liblzma/simple/simple_coder.h b/Utilities/cmliblzma/liblzma/simple/simple_coder.h index 0952fad..19c2ee0 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_coder.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_coder.h @@ -17,44 +17,56 @@ extern lzma_ret lzma_simple_x86_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_x86_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_powerpc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_powerpc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_ia64_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_ia64_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_arm_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_arm_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_armthumb_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/simple/simple_decoder.c b/Utilities/cmliblzma/liblzma/simple/simple_decoder.c index 034e158..1d864f2 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_decoder.c +++ b/Utilities/cmliblzma/liblzma/simple/simple_decoder.c @@ -14,18 +14,17 @@ extern lzma_ret -lzma_simple_props_decode(void **options, lzma_allocator *allocator, +lzma_simple_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_bcj *opt; - if (props_size == 0) return LZMA_OK; if (props_size != 4) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_bcj), allocator); + lzma_options_bcj *opt = lzma_alloc( + sizeof(lzma_options_bcj), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/simple/simple_decoder.h b/Utilities/cmliblzma/liblzma/simple/simple_decoder.h index b8bf590..bed8d37 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_decoder.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_decoder.h @@ -16,7 +16,7 @@ #include "simple_coder.h" extern lzma_ret lzma_simple_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/simple/simple_private.h b/Utilities/cmliblzma/liblzma/simple/simple_private.h index fcf9f7c..9d2c0fd 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_private.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_private.h @@ -16,9 +16,7 @@ #include "simple_coder.h" -typedef struct lzma_simple_s lzma_simple; - -struct lzma_coder_s { +typedef struct { /// Next filter in the chain lzma_next_coder next; @@ -33,12 +31,12 @@ struct lzma_coder_s { /// Pointer to filter-specific function, which does /// the actual filtering. - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size); /// Pointer to filter-specific data, or NULL if filter doesn't need /// any extra data. - lzma_simple *simple; + void *simple; /// The lowest 32 bits of the current position in the data. Most /// filters need this to do conversions between absolute and relative @@ -62,12 +60,13 @@ struct lzma_coder_s { /// Temporary buffer uint8_t buffer[]; -}; +} lzma_simple_coder; extern lzma_ret lzma_simple_coder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters, - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + const lzma_allocator *allocator, + const lzma_filter_info *filters, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size), size_t simple_size, size_t unfiltered_max, uint32_t alignment, bool is_encoder); diff --git a/Utilities/cmliblzma/liblzma/simple/sparc.c b/Utilities/cmliblzma/liblzma/simple/sparc.c index 0ddd2ac..74b2655 100644 --- a/Utilities/cmliblzma/liblzma/simple/sparc.c +++ b/Utilities/cmliblzma/liblzma/simple/sparc.c @@ -15,7 +15,7 @@ static size_t -sparc_code(lzma_simple *simple lzma_attribute((__unused__)), +sparc_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -26,8 +26,6 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), || (buffer[i] == 0x7F && (buffer[i + 1] & 0xC0) == 0xC0)) { - uint32_t dest; - uint32_t src = ((uint32_t)buffer[i + 0] << 24) | ((uint32_t)buffer[i + 1] << 16) | ((uint32_t)buffer[i + 2] << 8) @@ -35,6 +33,7 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), src <<= 2; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + src; else @@ -58,7 +57,7 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -sparc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +sparc_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -68,7 +67,8 @@ sparc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return sparc_coder_init(next, allocator, filters, true); } @@ -76,7 +76,8 @@ lzma_simple_sparc_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return sparc_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/x86.c b/Utilities/cmliblzma/liblzma/simple/x86.c index 95858e5..23d0c42 100644 --- a/Utilities/cmliblzma/liblzma/simple/x86.c +++ b/Utilities/cmliblzma/liblzma/simple/x86.c @@ -17,14 +17,14 @@ #define Test86MSByte(b) ((b) == 0 || (b) == 0xFF) -struct lzma_simple_s { +typedef struct { uint32_t prev_mask; uint32_t prev_pos; -}; +} lzma_simple_x86; static size_t -x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, +x86_code(void *simple_ptr, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { static const bool MASK_TO_ALLOWED_STATUS[8] @@ -33,39 +33,34 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, static const uint32_t MASK_TO_BIT_NUMBER[8] = { 0, 1, 2, 2, 3, 3, 3, 3 }; + lzma_simple_x86 *simple = simple_ptr; uint32_t prev_mask = simple->prev_mask; uint32_t prev_pos = simple->prev_pos; - size_t limit; - size_t buffer_pos; - if (size < 5) return 0; if (now_pos - prev_pos > 5) prev_pos = now_pos - 5; - limit = size - 5; - buffer_pos = 0; + const size_t limit = size - 5; + size_t buffer_pos = 0; while (buffer_pos <= limit) { - uint32_t offset; - uint32_t i; - uint8_t b = buffer[buffer_pos]; if (b != 0xE8 && b != 0xE9) { ++buffer_pos; continue; } - offset = now_pos + (uint32_t)(buffer_pos) + const uint32_t offset = now_pos + (uint32_t)(buffer_pos) - prev_pos; prev_pos = now_pos + (uint32_t)(buffer_pos); if (offset > 5) { prev_mask = 0; } else { - for (i = 0; i < offset; ++i) { + for (uint32_t i = 0; i < offset; ++i) { prev_mask &= 0x77; prev_mask <<= 1; } @@ -84,8 +79,6 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, uint32_t dest; while (true) { - uint32_t i; - if (is_encoder) dest = src + (now_pos + (uint32_t)( buffer_pos) + 5); @@ -96,7 +89,8 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, if (prev_mask == 0) break; - i = MASK_TO_BIT_NUMBER[prev_mask >> 1]; + const uint32_t i = MASK_TO_BIT_NUMBER[ + prev_mask >> 1]; b = (uint8_t)(dest >> (24 - i * 8)); @@ -130,15 +124,17 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, static lzma_ret -x86_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +x86_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { const lzma_ret ret = lzma_simple_coder_init(next, allocator, filters, - &x86_code, sizeof(lzma_simple), 5, 1, is_encoder); + &x86_code, sizeof(lzma_simple_x86), 5, 1, is_encoder); if (ret == LZMA_OK) { - next->coder->simple->prev_mask = 0; - next->coder->simple->prev_pos = (uint32_t)(-5); + lzma_simple_coder *coder = next->coder; + lzma_simple_x86 *simple = coder->simple; + simple->prev_mask = 0; + simple->prev_pos = (uint32_t)(-5); } return ret; @@ -146,7 +142,8 @@ x86_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_x86_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_x86_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return x86_coder_init(next, allocator, filters, true); @@ -154,7 +151,8 @@ lzma_simple_x86_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_x86_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_x86_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return x86_coder_init(next, allocator, filters, false); diff --git a/Utilities/cmlibrhash/librhash/rhash.c b/Utilities/cmlibrhash/librhash/rhash.c index ad6249b..34e1eb3 100644 --- a/Utilities/cmlibrhash/librhash/rhash.c +++ b/Utilities/cmlibrhash/librhash/rhash.c @@ -18,6 +18,8 @@ #define _LARGEFILE64_SOURCE #define _FILE_OFFSET_BITS 64 +#include "ustd.h" /* Need this first within CMake. */ + #include <string.h> /* memset() */ #include <stdlib.h> /* free() */ #include <stddef.h> /* ptrdiff_t */ diff --git a/Utilities/cmlibuv/CMakeLists.txt b/Utilities/cmlibuv/CMakeLists.txt index ba1638e..a503041 100644 --- a/Utilities/cmlibuv/CMakeLists.txt +++ b/Utilities/cmlibuv/CMakeLists.txt @@ -208,6 +208,22 @@ if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") ) endif() +if(CMAKE_SYSTEM_NAME STREQUAL "kFreeBSD") + list(APPEND uv_libraries + freebsd-glue + kvm + ) + list(APPEND uv_headers + include/uv-bsd.h + ) + list(APPEND uv_sources + src/unix/bsd-ifaddrs.c + src/unix/freebsd.c + src/unix/kqueue.c + src/unix/posix-hrtime.c + ) +endif() + if(CMAKE_SYSTEM_NAME STREQUAL "NetBSD") list(APPEND uv_libraries kvm |
