diff options
author | Yann Collet <cyan@fb.com> | 2018-01-14 05:09:02 (GMT) |
---|---|---|
committer | Yann Collet <cyan@fb.com> | 2018-01-14 05:09:02 (GMT) |
commit | 151a50e16e558b9619b979ef1c0a525acc13ed92 (patch) | |
tree | de2eace172cbef23f0f12392487104aafeee5960 | |
parent | 7628d9d15a399404d3ee4c0c07fcf0f6a2f39ede (diff) | |
parent | e0e29cf09e12aa8e98275a736e98fba7eed93b3f (diff) | |
download | lz4-151a50e16e558b9619b979ef1c0a525acc13ed92.zip lz4-151a50e16e558b9619b979ef1c0a525acc13ed92.tar.gz lz4-151a50e16e558b9619b979ef1c0a525acc13ed92.tar.bz2 |
Merge branch 'master' into coverity_scan
65 files changed, 5623 insertions, 3150 deletions
diff --git a/.gitattributes b/.gitattributes index 6212bd4..9eb12c0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -19,3 +19,6 @@ # Windows *.bat text eol=crlf *.cmd text eol=crlf + +# .travis.yml merging +.travis.yml merge=ours @@ -0,0 +1,16 @@ +Installation +============= + +``` +make +make install # this command may require root access +``` + +LZ4's `Makefile` supports standard [Makefile conventions], +including [staged installs], [redirection], or [command redefinition]. +It is compatible with parallel builds (`-j#`). + +[Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html +[staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html +[redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html +[command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html @@ -3,13 +3,9 @@ This repository uses 2 different licenses : - all other files use a GPLv2 license, unless explicitly stated otherwise Relevant license is reminded at the top of each source file, -and with the presence of COPYING or LICENSE file. +and with presence of COPYING or LICENSE file in associated directories. -This model emphasizes the fact that -only files in the `lib` directory are designed to be included into 3rd party projects. - -Other files, such as those from `programs` or `examples` directory, -are not intended to be compiled outside of their context. -They can serve as source of inspiration, -but they should not be copy/pasted into 3rd party projects, -as this scenario is not supported. +This model is selected to emphasize that +files in the `lib` directory are designed to be included into 3rd party applications, +while all other files, in `programs`, `tests` or `examples`, +receive more limited attention and support for such scenario. @@ -3,6 +3,8 @@ # Copyright (C) Yann Collet 2011-2016 # All rights reserved. # +# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets +# # BSD license # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: @@ -26,16 +28,10 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # You can contact the author at : -# - LZ4 source repository : https://github.com/Cyan4973/lz4 +# - LZ4 source repository : https://github.com/lz4/lz4 # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c # ################################################################ -DESTDIR ?= -PREFIX ?= /usr/local -VOID := /dev/null - -LIBDIR ?= $(PREFIX)/lib -INCLUDEDIR=$(PREFIX)/include LZ4DIR = lib PRGDIR = programs TESTDIR = tests @@ -44,62 +40,91 @@ EXDIR = examples # Define nul output ifneq (,$(filter Windows%,$(OS))) -EXT = .exe +EXT = .exe +VOID = nul else -EXT = +EXT = +VOID = /dev/null endif -.PHONY: default all lib lz4 clean test versionsTest examples +.PHONY: default +default: lib-release lz4-release -default: lib lz4-release +.PHONY: all +all: allmost manuals -all: - @$(MAKE) -C $(LZ4DIR) $@ - @$(MAKE) -C $(PRGDIR) $@ - @$(MAKE) -C $(TESTDIR) $@ - @$(MAKE) -C $(EXDIR) $@ +.PHONY: allmost +allmost: lib lz4 examples -lib: - @$(MAKE) -C $(LZ4DIR) +.PHONY: lib lib-release liblz4.a +lib lib-release liblz4.a: + @$(MAKE) -C $(LZ4DIR) $@ -lz4: +.PHONY: lz4 lz4-release +lz4 : liblz4.a +lz4-release : lib-release +lz4 lz4-release : @$(MAKE) -C $(PRGDIR) $@ @cp $(PRGDIR)/lz4$(EXT) . -lz4-release: - @$(MAKE) -C $(PRGDIR) - @cp $(PRGDIR)/lz4$(EXT) . +.PHONY: examples +examples: lib lz4 + $(MAKE) -C $(EXDIR) test + +.PHONY: manuals +manuals: + @$(MAKE) -C contrib/gen_manual $@ +.PHONY: clean clean: + @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) @$(MAKE) -C $(TESTDIR) $@ > $(VOID) - @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(MAKE) -C $(EXDIR) $@ > $(VOID) - @$(MAKE) -C examples $@ > $(VOID) + @$(MAKE) -C contrib/gen_manual $@ > $(VOID) @$(RM) lz4$(EXT) @echo Cleaning completed -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#----------------------------------------------------------------------------- +# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) HOST_OS = POSIX -install: - @$(MAKE) -C $(LZ4DIR) $@ - @$(MAKE) -C $(PRGDIR) $@ - -uninstall: +.PHONY: install uninstall +install uninstall: @$(MAKE) -C $(LZ4DIR) $@ @$(MAKE) -C $(PRGDIR) $@ travis-install: - $(MAKE) install PREFIX=~/install_test_dir + $(MAKE) -j1 install DESTDIR=~/install_test_dir + +cmake: + @cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE) + +endif + + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +CMAKE_PARAMS = -G"MSYS Makefiles" +endif + +#------------------------------------------------------------------------ +#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) + +.PHONY: list +list: + @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs + +.PHONY: test test: - $(MAKE) -C $(TESTDIR) test + $(MAKE) -C $(TESTDIR) $@ clangtest: clean clang -v @@ -130,31 +155,10 @@ platformTest: clean CFLAGS="-O3 -Werror -static" $(MAKE) -C $(TESTDIR) all $(MAKE) -C $(TESTDIR) test-platform +.PHONY: versionsTest versionsTest: clean $(MAKE) -C $(TESTDIR) $@ -examples: - $(MAKE) -C $(LZ4DIR) - $(MAKE) -C $(PRGDIR) lz4 - $(MAKE) -C examples test - -endif - - -ifneq (,$(filter MSYS%,$(shell uname))) -HOST_OS = MSYS -CMAKE_PARAMS = -G"MSYS Makefiles" -endif - - -#------------------------------------------------------------------------ -#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets -#------------------------------------------------------------------------ -ifneq (,$(filter $(HOST_OS),MSYS POSIX)) - -cmake: - @cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE) - gpptest: clean g++ -v CC=g++ $(MAKE) -C $(LZ4DIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" @@ -168,13 +172,10 @@ gpptest32: clean CC=g++ $(MAKE) -C $(TESTDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" c_standards: clean - $(MAKE) all MOREFLAGS="-std=gnu90 -Werror" - $(MAKE) clean - $(MAKE) all MOREFLAGS="-std=c99 -Werror" - $(MAKE) clean - $(MAKE) all MOREFLAGS="-std=gnu99 -Werror" - $(MAKE) clean - $(MAKE) all MOREFLAGS="-std=c11 -Werror" - $(MAKE) clean + # note : lz4 is not C90 compatible, because it requires long long support + CFLAGS="-std=gnu90 -Werror" $(MAKE) clean allmost + CFLAGS="-std=c99 -Werror" $(MAKE) clean allmost + CFLAGS="-std=gnu99 -Werror" $(MAKE) clean allmost + CFLAGS="-std=c11 -Werror" $(MAKE) clean allmost endif @@ -1,7 +1,47 @@ +v1.8.1 +perf : faster and stronger ultra modes (levels 10+) +perf : slightly faster compression and decompression speed +perf : fix bad degenerative case, reported by @c-morgenstern +fix : decompression failed when using a combination of extDict + low memory address (#397), reported and fixed by Julian Scheid (@jscheid) +cli : support for dictionary compression (`-D`), by Felix Handte @felixhandte +cli : fix : `lz4 -d --rm` preserves timestamp (#441) +cli : fix : do not modify /dev/null permission as root, by @aliceatlas +api : `_destSize()` variant supported for all compression levels +build : `make` and `make test` compatible with `-jX`, reported by @mwgamera +build : can control LZ4LIB_VISIBILITY macro, by @mikir +install: fix man page directory (#387), reported by Stuart Cardall (@itoffshore) + +v1.8.0 +cli : fix : do not modify /dev/null permissions, reported by @Maokaman1 +cli : added GNU separator -- specifying that all following arguments are files +API : added LZ4_compress_HC_destSize(), by Oleg (@remittor) +API : added LZ4F_resetDecompressionContext() +API : lz4frame : negative compression levels trigger fast acceleration, request by Lawrence Chan +API : lz4frame : can control block checksum and dictionary ID +API : fix : expose obsolete decoding functions, reported by Chen Yufei +API : experimental : lz4frame_static : new dictionary compression API +build : fix : static lib installation, by Ido Rosen +build : dragonFlyBSD, OpenBSD, NetBSD supported +build : LZ4_MEMORY_USAGE can be modified at compile time, through external define +doc : Updated LZ4 Frame format to v1.6.0, restoring Dictionary-ID field +doc : lz4 api manual, by Przemyslaw Skibinski + +v1.7.5 +lz4hc : new high compression mode : levels 10-12 compress more and slower, by Przemyslaw Skibinski +lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) +cli : fix minor notification when using -r recursive mode +API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) +doc : markdown version of man page, by Takayuki Matsuoka (#279) +build : Makefile : fix make -jX lib+exe concurrency (#277) +build : cmake : improvements by Michał Górny (#296) + +v1.7.4.2 +fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) + v1.7.4 Improved : much better speed in -mx32 mode cli : fix : Large file support in 32-bits mode on Mac OS-X -fix : compilation on gcc 4.4 (#282), reported by Antoine Martin +fix : compilation on gcc 4.4 (#272), reported by Antoine Martin v1.7.3 Changed : moved to versioning; package, cli and library have same version number @@ -25,14 +25,13 @@ LZ4 library is provided as open-source software using BSD 2-Clause license. [travisMasterBadge]: https://travis-ci.org/lz4/lz4.svg?branch=master "Continuous Integration test suite" [travisDevBadge]: https://travis-ci.org/lz4/lz4.svg?branch=dev "Continuous Integration test suite" [travisLink]: https://travis-ci.org/lz4/lz4 -[AppveyorMasterBadge]: https://ci.appveyor.com/api/projects/status/jc2yhgwyc7qqtsko/branch/master?svg=true "Windows test suite" -[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/jc2yhgwyc7qqtsko/branch/dev?svg=true "Windows test suite" +[AppveyorMasterBadge]: https://ci.appveyor.com/api/projects/status/github/lz4/lz4?branch=master&svg=true "Windows test suite" +[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/github/lz4/lz4?branch=dev&svg=true "Windows test suite" [AppveyorLink]: https://ci.appveyor.com/project/YannCollet/lz4-1lndh [coverBadge]: https://scan.coverity.com/projects/4735/badge.svg "Static code analysis of Master branch" [coverlink]: https://scan.coverity.com/projects/4735 > **Branch Policy:** - > - The "master" branch is considered stable, at all times. > - The "dev" branch is the one where all contributions must be merged before being promoted to master. @@ -70,7 +69,25 @@ in single-thread mode. [zlib]: http://www.zlib.net/ [Zstandard]: http://www.zstd.net/ -LZ4 is also compatible and well optimized for x32 mode, for which it provides +10% speed performance. +LZ4 is also compatible and well optimized for x32 mode, for which it provides an additional +10% speed performance. + + +Installation +------------------------- + +``` +make +make install # this command may require root access +``` + +LZ4's `Makefile` supports standard [Makefile conventions], +including [staged installs], [redirection], or [command redefinition]. +It is compatible with parallel builds (`-j#`). + +[Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html +[staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html +[redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html +[command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html Documentation diff --git a/appveyor.yml b/appveyor.yml index 93c1101..056719a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -82,7 +82,7 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /p:EnableWholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && ECHO *** && diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..9a0d1ec --- /dev/null +++ b/circle.yml @@ -0,0 +1,38 @@ +dependencies: + override: + - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update + - sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu + - sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross + - sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-5-multilib gcc-6 valgrind + +test: + override: + # Tests compilers and C standards + - clang -v; make clangtest && make clean + - g++ -v; make gpptest && make clean + - gcc -v; make c_standards && make clean + - gcc-5 -v; make -C tests test-lz4 CC=gcc-5 MOREFLAGS=-Werror && make clean + - gcc-5 -v; make -C tests test-lz4c32 CC=gcc-5 MOREFLAGS="-I/usr/include/x86_64-linux-gnu -Werror" && make clean + - gcc-6 -v; make c_standards CC=gcc-6 && make clean + - gcc-6 -v; make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean +# Shorter tests + - make cmake && make clean + - make -C tests test-lz4 + - make -C tests test-lz4c + - make -C tests test-frametest + - make -C tests test-fullbench + - make -C tests test-fuzzer && make clean + - make -C lib all && make clean + - pyenv global 3.4.4; CFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean + - make travis-install && make clean + # Longer tests + - gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean + - make usan && make clean + - clang -v; make staticAnalyze && make clean + # Valgrind tests + - make -C tests test-mem && make clean + # ARM, AArch64, PowerPC, PowerPC64 tests + - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make clean + - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 && make clean + - make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make clean + - make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static && make clean diff --git a/contrib/cmake_unofficial/.gitignore b/contrib/cmake_unofficial/.gitignore index 0f81929..d39505d 100644 --- a/contrib/cmake_unofficial/.gitignore +++ b/contrib/cmake_unofficial/.gitignore @@ -5,3 +5,5 @@ CMakeFiles *.cmake Makefile liblz4.pc +lz4c +install_manifest.txt diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index de070d6..27c3a78 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -65,6 +65,11 @@ endif(NOT LZ4_BUNDLED_MODE AND NOT CPack_CMake_INCLUDED) # which case we always use static libraries. include(CMakeDependentOption) CMAKE_DEPENDENT_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON "NOT LZ4_BUNDLED_MODE" OFF) +CMAKE_DEPENDENT_OPTION(BUILD_STATIC_LIBS "Build static libraries" OFF "BUILD_SHARED_LIBS" ON) + +if(NOT BUILD_SHARED_LIBS AND NOT BUILD_STATIC_LIBS) + message(FATAL_ERROR "Both BUILD_SHARED_LIBS and BUILD_STATIC_LIBS have been disabled") +endif() set(LZ4_LIB_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/lib") set(LZ4_PROG_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/programs") @@ -90,28 +95,41 @@ set(LZ4_CLI_SOURCES # we're building a shared library this is ignored and PIC is always # used. option(LZ4_POSITION_INDEPENDENT_LIB "Use position independent code for static library (if applicable)" ON) -if(LZ4_POSITION_INDEPENDENT_LIB OR BUILD_SHARED_LIBS) - set(LZ4_POSITION_INDEPENDENT_CODE TRUE) -else() - set(LZ4_POSITION_INDEPENDENT_CODE FALSE) -endif() # liblz4 -add_library(lz4 ${LZ4_SOURCES}) -set_target_properties(lz4 PROPERTIES - SOVERSION "${LZ4_VERSION_STRING}" - VERSION "${LZ4_VERSION_STRING}" - POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_CODE}) +set(LZ4_LIBRARIES_BUILT) +if(BUILD_SHARED_LIBS) + add_library(lz4_shared SHARED ${LZ4_SOURCES}) + set_target_properties(lz4_shared PROPERTIES + OUTPUT_NAME lz4 + SOVERSION "${LZ4_VERSION_MAJOR}" + VERSION "${LZ4_VERSION_STRING}") + list(APPEND LZ4_LIBRARIES_BUILT lz4_shared) +endif() +if(BUILD_STATIC_LIBS) + add_library(lz4_static STATIC ${LZ4_SOURCES}) + set_target_properties(lz4_static PROPERTIES + OUTPUT_NAME lz4 + POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_LIB}) + list(APPEND LZ4_LIBRARIES_BUILT lz4_static) +endif() + +# link to shared whenever possible, to static otherwise +if(BUILD_SHARED_LIBS) + set(LZ4_LINK_LIBRARY lz4_shared) +else() + set(LZ4_LINK_LIBRARY lz4_static) +endif() # lz4 add_executable(lz4cli ${LZ4_CLI_SOURCES}) set_target_properties(lz4cli PROPERTIES OUTPUT_NAME lz4) -target_link_libraries(lz4cli lz4) +target_link_libraries(lz4cli ${LZ4_LINK_LIBRARY}) # lz4c add_executable(lz4c ${LZ4_CLI_SOURCES}) set_target_properties(lz4c PROPERTIES COMPILE_DEFINITIONS "ENABLE_LZ4C_LEGACY_OPTIONS") -target_link_libraries(lz4c lz4) +target_link_libraries(lz4c ${LZ4_LINK_LIBRARY}) # Extra warning flags include (CheckCCompilerFlag) @@ -149,16 +167,38 @@ if(NOT LZ4_BUNDLED_MODE) install(TARGETS lz4cli lz4c RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") - install(TARGETS lz4 + install(TARGETS ${LZ4_LIBRARIES_BUILT} LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" - ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" + RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") install(FILES "${LZ4_LIB_SOURCE_DIR}/lz4.h" "${LZ4_LIB_SOURCE_DIR}/lz4frame.h" "${LZ4_LIB_SOURCE_DIR}/lz4hc.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + install(FILES "${LZ4_PROG_SOURCE_DIR}/lz4.1" + DESTINATION "${CMAKE_INSTALL_MANDIR}/man1") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblz4.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") + + # install lz4cat and unlz4 symlinks on *nix + if(UNIX) + install(CODE " + foreach(f lz4cat unlz4) + set(dest \"\$ENV{DESTDIR}${CMAKE_INSTALL_FULL_BINDIR}/\${f}\") + message(STATUS \"Symlinking: \${dest} -> lz4\") + execute_process( + COMMAND \"${CMAKE_COMMAND}\" -E create_symlink lz4 \"\${dest}\") + endforeach() + ") + + # create manpage aliases + foreach(f lz4cat unlz4) + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${f}.1" ".so man1/lz4.1\n") + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${f}.1" + DESTINATION "${CMAKE_INSTALL_MANDIR}/man1") + endforeach() + endif(UNIX) endif(NOT LZ4_BUNDLED_MODE) # pkg-config @@ -176,4 +216,6 @@ else() set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}") endif() +# for liblz4.pc substitution +set(VERSION ${LZ4_VERSION_STRING}) configure_file(${LZ4_LIB_SOURCE_DIR}/liblz4.pc.in liblz4.pc @ONLY) diff --git a/contrib/gen_manual/.gitignore b/contrib/gen_manual/.gitignore new file mode 100644 index 0000000..6ea967f --- /dev/null +++ b/contrib/gen_manual/.gitignore @@ -0,0 +1,2 @@ +# build artefact +gen_manual diff --git a/contrib/gen_manual/Makefile b/contrib/gen_manual/Makefile index adbcca2..9fbe858 100644 --- a/contrib/gen_manual/Makefile +++ b/contrib/gen_manual/Makefile @@ -35,7 +35,15 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -W CFLAGS += $(MOREFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) - +LZ4API = ../../lib/lz4.h +LZ4MANUAL = ../../doc/lz4_manual.html +LZ4FAPI = ../../lib/lz4frame.h +LZ4FMANUAL = ../../doc/lz4frame_manual.html +LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)` +LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)` +LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LZ4VER := $(shell echo $(LIBVER_SCRIPT)) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -45,14 +53,24 @@ EXT = endif -.PHONY: default gen_manual - +.PHONY: default default: gen_manual gen_manual: gen_manual.cpp - $(CXX) $(FLAGS) $^ -o $@$(EXT) + $(CXX) $(FLAGS) $^ -o $@$(EXT) + +$(LZ4MANUAL) : gen_manual $(LZ4API) + echo "Update lz4 manual in /doc" + ./gen_manual $(LZ4VER) $(LZ4API) $@ + +$(LZ4FMANUAL) : gen_manual $(LZ4FAPI) + echo "Update lz4frame manual in /doc" + ./gen_manual $(LZ4VER) $(LZ4FAPI) $@ +.PHONY: manuals +manuals: gen_manual $(LZ4MANUAL) $(LZ4FMANUAL) +.PHONY: clean clean: @$(RM) gen_manual$(EXT) @echo Cleaning completed diff --git a/contrib/gen_manual/gen-lz4-manual.sh b/contrib/gen_manual/gen-lz4-manual.sh index 55d31a4..73a7214 100644 --- a/contrib/gen_manual/gen-lz4-manual.sh +++ b/contrib/gen_manual/gen-lz4-manual.sh @@ -6,4 +6,5 @@ LIBVER_PATCH_SCRIPT=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][ LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT echo LZ4_VERSION=$LIBVER_SCRIPT -./gen_manual $LIBVER_SCRIPT ../../lib/lz4.h ./lz4_manual.html +./gen_manual "lz4 $LIBVER_SCRIPT" ../../lib/lz4.h ./lz4_manual.html +./gen_manual "lz4frame $LIBVER_SCRIPT" ../../lib/lz4frame.h ./lz4frame_manual.html diff --git a/contrib/gen_manual/gen_manual.cpp b/contrib/gen_manual/gen_manual.cpp index 2df081d..65abd3a 100644 --- a/contrib/gen_manual/gen_manual.cpp +++ b/contrib/gen_manual/gen_manual.cpp @@ -89,11 +89,13 @@ vector<string> get_lines(vector<string>& input, int& linenum, string terminator) /* print line with LZ4LIB_API removed and C++ comments not bold */ void print_line(stringstream &sout, string line) { - size_t spos; + size_t spos, epos; if (line.substr(0,11) == "LZ4LIB_API ") line = line.substr(11); + if (line.substr(0,12) == "LZ4FLIB_API ") line = line.substr(12); spos = line.find("/*"); - if (spos!=string::npos) { + epos = line.find("*/"); + if (spos!=string::npos && epos!=string::npos) { sout << line.substr(0, spos); sout << "</b>" << line.substr(spos) << "<b>" << endl; } else { @@ -118,7 +120,7 @@ int main(int argc, char *argv[]) { return 1; } - version = "lz4 " + string(argv[1]) + " Manual"; + version = string(argv[1]) + " Manual"; istream.open(argv[2], ifstream::in); if (!istream.is_open()) { @@ -158,36 +160,28 @@ int main(int argc, char *argv[]) { continue; } - /* comments of type /*= and /**= mean: use a <H3> header and show also all functions until first empty line */ - if ((line.substr(0,3) == "/*=" || line.substr(0,4) == "/**=") && line.find("*/")!=string::npos) { - trim_comments(line); - trim(line, "= "); - sout << "<h3>" << line << "</h3><pre><b>"; - lines = get_lines(input, ++linenum, ""); - for (l=0; l<lines.size(); l++) { - print_line(sout, lines[l]); - } - sout << "</b></pre><BR>" << endl; - continue; + spos = line.find("/**="); + if (spos==string::npos) { + spos = line.find("/*!"); + if (spos==string::npos) + spos = line.find("/**"); + if (spos==string::npos) + spos = line.find("/*-"); + if (spos==string::npos) + spos = line.find("/*="); + if (spos==string::npos) + continue; + exclam = line[spos+2]; } + else exclam = '='; - spos = line.find("/*!"); - if (spos==string::npos) - spos = line.find("/**"); - if (spos==string::npos) - spos = line.find("/*-"); - - if (spos==string::npos) - continue; - - exclam = line[spos+2]; comments = get_lines(input, linenum, "*/"); if (!comments.empty()) comments[0] = line.substr(spos+3); if (!comments.empty()) comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/")); for (l=0; l<comments.size(); l++) { if (comments[l].find(" *")==0) comments[l] = comments[l].substr(2); else if (comments[l].find(" *")==0) comments[l] = comments[l].substr(3); - trim(comments[l], "*-"); + trim(comments[l], "*-="); } while (!comments.empty() && comments[comments.size()-1].empty()) comments.pop_back(); // remove empty line at the end while (!comments.empty() && comments[0].empty()) comments.erase(comments.begin()); // remove empty line at the start @@ -208,6 +202,18 @@ int main(int argc, char *argv[]) { print_line(sout, comments[l]); } sout << "</p></pre><BR>" << endl << endl; + } else if (exclam == '=') { /* comments of type /*= and /**= mean: use a <H3> header and show also all functions until first empty line */ + trim(comments[0], " "); + sout << "<h3>" << comments[0] << "</h3><pre>"; + for (l=1; l<comments.size(); l++) { + print_line(sout, comments[l]); + } + sout << "</pre><b><pre>"; + lines = get_lines(input, ++linenum, ""); + for (l=0; l<lines.size(); l++) { + print_line(sout, lines[l]); + } + sout << "</pre></b><BR>" << endl; } else { /* comments of type /** and /*- mean: this is a comment; use a <H2> header for the first line */ if (comments.empty()) continue; @@ -238,4 +244,4 @@ int main(int argc, char *argv[]) { ostream << "</html>" << endl << "</body>" << endl; return 0; -} +}
\ No newline at end of file diff --git a/doc/lz4_Block_format.md b/doc/lz4_Block_format.md index 0f6a5ba..4e39b41 100644 --- a/doc/lz4_Block_format.md +++ b/doc/lz4_Block_format.md @@ -90,10 +90,18 @@ A 255 value means there is another byte to read and add. There is no limit to the number of optional bytes that can be output this way. (This points towards a maximum achievable compression ratio of about 250). -With the offset and the matchlength, -the decoder can now proceed to copy the data from the already decoded buffer. -On decoding the matchlength, we reach the end of the compressed sequence, -and therefore start another one. +Decoding the matchlength reaches the end of current sequence. +Next byte will be the start of another sequence. +But before moving to next sequence, +it's time to use the decoded match position and length. +The decoder copies matchlength bytes from match position to current position. + +In some cases, matchlength is larger than offset. +Therefore, match pos + match length > current pos, +which means that later bytes to copy are not yet decoded. +This is called an "overlap match", and must be handled with special care. +The most common case is an offset of 1, +meaning the last byte is repeated matchlength times. Parsing restrictions diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md index 2ea1a86..77454b2 100644 --- a/doc/lz4_Frame_format.md +++ b/doc/lz4_Frame_format.md @@ -1,7 +1,7 @@ LZ4 Frame Format Description ============================ -###Notices +### Notices Copyright (c) 2013-2015 Yann Collet @@ -14,9 +14,9 @@ and that any substantive changes or deletions from the original are clearly marked. Distribution of this document is unlimited. -###Version +### Version -1.5.1 (31/03/2015) +1.6.0 (08/08/2017) Introduction @@ -63,7 +63,7 @@ General Structure of LZ4 Frame format | MagicNb | F. Descriptor | Block | (...) | EndMark | C. Checksum | |:-------:|:-------------:| ----- | ----- | ------- | ----------- | -| 4 bytes | 3-11 bytes | | | 4 bytes | 0-4 bytes | +| 4 bytes | 3-15 bytes | | | 4 bytes | 0-4 bytes | __Magic Number__ @@ -72,7 +72,7 @@ Value : 0x184D2204 __Frame Descriptor__ -3 to 11 Bytes, to be detailed in the next part. +3 to 15 Bytes, to be detailed in the next part. Most important part of the spec. __Data Blocks__ @@ -118,31 +118,31 @@ to decode all concatenated frames in their sequential order. Frame Descriptor ---------------- -| FLG | BD | (Content Size) | HC | -| ------- | ------- |:--------------:| ------- | -| 1 byte | 1 byte | 0 - 8 bytes | 1 byte | +| FLG | BD | (Content Size) | (Dictionary ID) | HC | +| ------- | ------- |:--------------:|:---------------:| ------- | +| 1 byte | 1 byte | 0 - 8 bytes | 0 - 4 bytes | 1 byte | The descriptor uses a minimum of 3 bytes, -and up to 11 bytes depending on optional parameters. +and up to 15 bytes depending on optional parameters. __FLG byte__ -| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1-0 | -| ------- | ------- | ------- | --------- | ------- | --------- | -------- | -|FieldName| Version | B.Indep | B.Checksum| C.Size | C.Checksum|*Reserved*| +| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1 | 0 | +| ------- |-------|-------|----------|------|----------|----------|------| +|FieldName|Version|B.Indep|B.Checksum|C.Size|C.Checksum|*Reserved*|DictID| __BD byte__ -| BitNb | 7 | 6-5-4 | 3-2-1-0 | -| ------- | -------- | ------------ | -------- | -|FieldName|*Reserved*| Block MaxSize|*Reserved*| +| BitNb | 7 | 6-5-4 | 3-2-1-0 | +| ------- | -------- | ------------- | -------- | +|FieldName|*Reserved*| Block MaxSize |*Reserved*| In the tables, bit 7 is highest bit, while bit 0 is lowest. __Version Number__ -2-bits field, must be set to “01”. +2-bits field, must be set to `01`. Any other value cannot be decoded by this version of the specification. Other version numbers will use different flag layouts. @@ -154,7 +154,7 @@ If this flag is set to “0”, each block depends on previous ones In such case, it’s necessary to decode all blocks in sequence. Block dependency improves compression ratio, especially for small blocks. -On the other hand, it makes direct jumps or multi-threaded decoding impossible. +On the other hand, it makes random access or multi-threaded decoding impossible. __Block checksum flag__ @@ -172,13 +172,17 @@ Content Size usage is optional. __Content checksum flag__ -If this flag is set, a content checksum will be appended after the EndMark. +If this flag is set, a 32-bits content checksum will be appended +after the EndMark. -Recommended value : “1” (content checksum is present) +__Dictionary ID flag__ + +If this flag is set, a 4-bytes Dict-ID field will be present, +after the descriptor flags and the Content Size. __Block Maximum Size__ -This information is intended to help the decoder allocate memory. +This information is useful to help the decoder allocate memory. Size here refers to the original (uncompressed) data size. Block Maximum Size is one value among the following table : @@ -186,17 +190,17 @@ Block Maximum Size is one value among the following table : | --- | --- | --- | --- | ----- | ------ | ---- | ---- | | N/A | N/A | N/A | N/A | 64 KB | 256 KB | 1 MB | 4 MB | -The decoder may refuse to allocate block sizes above a (system-specific) size. +The decoder may refuse to allocate block sizes above any system-specific size. Unused values may be used in a future revision of the spec. -A decoder conformant to the current version of the spec -is only able to decode blocksizes defined in this spec. +A decoder conformant with the current version of the spec +is only able to decode block sizes defined in this spec. __Reserved bits__ Value of reserved bits **must** be 0 (zero). Reserved bit might be used in a future version of the specification, typically enabling new optional features. -If this happens, a decoder respecting the current version of the specification +When this happens, a decoder respecting the current specification version shall not be able to decode such a frame. __Content Size__ @@ -208,12 +212,32 @@ Format is Little endian. This value is informational, typically for display or memory allocation. It can be skipped by a decoder, or used to validate content correctness. +__Dictionary ID__ + +Dict-ID is only present if the associated flag is set. +It's an unsigned 32-bits value, stored using little-endian convention. +A dictionary is useful to compress short input sequences. +The compressor can take advantage of the dictionary context +to encode the input in a more compact manner. +It works as a kind of “known prefix” which is used by +both the compressor and the decompressor to “warm-up” reference tables. + +The decompressor can use Dict-ID identifier to determine +which dictionary must be used to correctly decode data. +The compressor and the decompressor must use exactly the same dictionary. +It's presumed that the 32-bits dictID uniquely identifies a dictionary. + +Within a single frame, a single dictionary can be defined. +When the frame descriptor defines independent blocks, +each block will be initialized with the same dictionary. +If the frame descriptor defines linked blocks, +the dictionary will only be used once, at the beginning of the frame. + __Header Checksum__ One-byte checksum of combined descriptor fields, including optional ones. -The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF ` -using zero as a seed, -and the full Frame Descriptor as an input +The value is the second byte of `xxh32()` : ` (xxh32()>>8) & 0xFF ` +using zero as a seed, and the full Frame Descriptor as an input (including optional fields when they are present). A wrong checksum indicates an error in the descriptor. Header checksum is informational and can be skipped. @@ -347,7 +371,7 @@ wether it is a file or a stream. Alternatively, if the frame is followed by a valid Frame Magic Number, it is considered completed. -It makes legacy frames compatible with frame concatenation. +This policy makes it possible to concatenate legacy frames. Any other value will be interpreted as a block size, and trigger an error if it does not fit within acceptable range. @@ -356,7 +380,9 @@ and trigger an error if it does not fit within acceptable range. Version changes --------------- -1.5.1 : changed format to MarkDown compatible +1.6.0 : restored Dictionary ID field in Frame header + +1.5.1 : changed document format to MarkDown 1.5 : removed Dictionary ID from specification diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index bc46645..6b7935d 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -1,20 +1,22 @@ <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"> -<title>lz4 1.7.2 Manual</title> +<title>1.8.1 Manual</title> </head> <body> -<h1>lz4 1.7.2 Manual</h1> +<h1>1.8.1 Manual</h1> <hr> <a name="Contents"></a><h2>Contents</h2> <ol> <li><a href="#Chapter1">Introduction</a></li> -<li><a href="#Chapter2">Tuning parameter</a></li> -<li><a href="#Chapter3">Private definitions</a></li> +<li><a href="#Chapter2">Version</a></li> +<li><a href="#Chapter3">Tuning parameter</a></li> <li><a href="#Chapter4">Simple Functions</a></li> <li><a href="#Chapter5">Advanced Functions</a></li> <li><a href="#Chapter6">Streaming Compression Functions</a></li> <li><a href="#Chapter7">Streaming Decompression Functions</a></li> +<li><a href="#Chapter8">Private definitions</a></li> +<li><a href="#Chapter9">Obsolete Functions</a></li> </ol> <hr> <a name="Chapter1"></a><h2>Introduction</h2><pre> @@ -29,19 +31,26 @@ - unbounded multiple steps (described as Streaming compression) lz4.h provides block compression functions. It gives full buffer control to user. - Block compression functions are not-enough to send information, - since it's still necessary to provide metadata (such as compressed size), - and each application can do it in whichever way it wants. - For interoperability, there is LZ4 frame specification (doc/lz4_Frame_format.md). + Decompressing an lz4-compressed block also requires metadata (such as compressed size). + Each application is free to encode such metadata in whichever way it wants. + + An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), + take care of encoding standard metadata alongside LZ4-compressed blocks. + If your application requires interoperability, it's recommended to use it. A library is provided to take care of it, see lz4frame.h. <BR></pre> -<h3>Version</h3><pre><b>int LZ4_versionNumber (void); -const char* LZ4_versionString (void); +<a name="Chapter2"></a><h2>Version</h2><pre></pre> + +<pre><b>int LZ4_versionNumber (void); </b>/**< library version number; to be used when checking dll version */<b> </b></pre><BR> -<a name="Chapter2"></a><h2>Tuning parameter</h2><pre></pre> +<pre><b>const char* LZ4_versionString (void); </b>/**< library version string; to be used when checking dll version */<b> +</b></pre><BR> +<a name="Chapter3"></a><h2>Tuning parameter</h2><pre></pre> -<pre><b>#define LZ4_MEMORY_USAGE 14 +<pre><b>#ifndef LZ4_MEMORY_USAGE +# define LZ4_MEMORY_USAGE 14 +#endif </b><p> Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) Increasing memory usage improves compression ratio Reduced memory usage can improve speed, due to cache effect @@ -49,66 +58,28 @@ const char* LZ4_versionString (void); </p></pre><BR> -<a name="Chapter3"></a><h2>Private definitions</h2><pre> - Do not use these definitions. - They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - If you use these definitions in your code, it will break when you upgrade LZ4 to a new version. -<BR></pre> - -<pre><b>typedef struct { - uint32_t hashTable[LZ4_HASH_SIZE_U32]; - uint32_t currentOffset; - uint32_t initCheck; - const uint8_t* dictionary; - uint8_t* bufferStart; </b>/* obsolete, used for slideInputBuffer */<b> - uint32_t dictSize; -} LZ4_stream_t_internal; -</b></pre><BR> -<pre><b>typedef struct { - const uint8_t* externalDict; - size_t extDictSize; - const uint8_t* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; -</b></pre><BR> -<pre><b>typedef struct { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned int initCheck; - const unsigned char* dictionary; - unsigned char* bufferStart; </b>/* obsolete, used for slideInputBuffer */<b> - unsigned int dictSize; -} LZ4_stream_t_internal; -</b></pre><BR> -<pre><b>typedef struct { - const unsigned char* externalDict; - size_t extDictSize; - const unsigned char* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; -</b></pre><BR> <a name="Chapter4"></a><h2>Simple Functions</h2><pre></pre> -<pre><b>int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); -</b><p> Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). +<pre><b>int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); +</b><p> Compresses 'srcSize' bytes from buffer 'src' + into already allocated 'dst' buffer of size 'dstCapacity'. + Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, + If the function cannot compress 'src' into a limited 'dst' budget, compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails + As a consequence, 'dst' content is not valid. + This function never writes outside 'dst' buffer, nor read outside 'source' buffer. + srcSize : supported max value is LZ4_MAX_INPUT_VALUE + dstCapacity : full or partial size of buffer 'dst' (which must be already allocated) + return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + or 0 if compression fails </p></pre><BR> -<pre><b>int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); -</b><p> compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). +<pre><b>int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); +</b><p> compressedSize : is the exact complete size of the compressed block. + dstCapacity : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + If destination buffer is not large enough, decoding will stop and output an error code (negative value). If the source stream is detected malformed, the function will stop decoding and return a negative result. This function is protected against buffer overflow exploits, including malicious data packets. It never writes outside output buffer, nor reads outside input buffer. @@ -126,7 +97,7 @@ const char* LZ4_versionString (void); or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) </p></pre><BR> -<pre><b>int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); +<pre><b>int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); </b><p> Same as LZ4_compress_default(), but allows to select an "acceleration" factor. The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. @@ -135,153 +106,231 @@ const char* LZ4_versionString (void); </p></pre><BR> <pre><b>int LZ4_sizeofState(void); -int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); +int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); </b><p> Same compression function, just using an externally allocated memory space to store compression state. Use LZ4_sizeofState() to know how much memory must be allocated, and allocate it on 8-bytes boundaries (using malloc() typically). Then, provide it as 'void* state' to compression function. </p></pre><BR> -<pre><b>int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); -</b><p> Reverse the logic, by compressing as much data as possible from 'source' buffer - into already allocated buffer 'dest' of size 'targetDestSize'. - This function either compresses the entire 'source' content into 'dest' if it's large enough, - or fill 'dest' buffer completely with as much data as possible from 'source'. - *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. - New value is necessarily <= old value. - return : Nb bytes written into 'dest' (necessarily <= targetDestSize) - or 0 if compression fails +<pre><b>int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); +</b><p> Reverse the logic : compresses as much data as possible from 'src' buffer + into already allocated buffer 'dst' of size 'targetDestSize'. + This function either compresses the entire 'src' content into 'dst' if it's large enough, + or fill 'dst' buffer completely with as much data as possible from 'src'. + *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + New value is necessarily <= old value. + return : Nb bytes written into 'dst' (necessarily <= targetDestSize) + or 0 if compression fails </p></pre><BR> -<pre><b>int LZ4_decompress_fast (const char* source, char* dest, int originalSize); -</b><p> originalSize : is the original and therefore uncompressed size +<pre><b>int LZ4_decompress_fast (const char* src, char* dst, int originalSize); +</b><p> originalSize : is the original uncompressed size return : the number of bytes read from the source buffer (in other words, the compressed size) If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. + Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes. + note : This function respects memory boundaries for *properly formed* compressed data. It is a bit faster than LZ4_decompress_safe(). However, it does not provide any protection against intentionally modified data stream (malicious input). Use this function in trusted environment only (data to decode comes from a trusted source). </p></pre><BR> -<pre><b>int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); -</b><p> This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. +<pre><b>int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); +</b><p> This function decompress a compressed block of size 'srcSize' at position 'src' + into destination buffer 'dst' of size 'dstCapacity'. + The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that. + However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity). + @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity) + Note : this number can be < 'targetOutputSize' should the compressed block contain less data. Always control how many bytes were decoded. If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets. </p></pre><BR> <a name="Chapter6"></a><h2>Streaming Compression Functions</h2><pre></pre> -<pre><b>typedef struct { - union { - long long table[LZ4_STREAMSIZE_U64]; - LZ4_stream_t_internal internal_donotuse; - }; -} LZ4_stream_t; -</b><p> information structure to track an LZ4 stream. - important : init this structure content before first use ! - note : only allocated directly the structure if you are statically linking LZ4 - If you are using liblz4 as a DLL, please use below construction methods instead. - -</p></pre><BR> - -<pre><b>void LZ4_resetStream (LZ4_stream_t* streamPtr); -</b><p> Use this function to init an allocated `LZ4_stream_t` structure - -</p></pre><BR> - <pre><b>LZ4_stream_t* LZ4_createStream(void); int LZ4_freeStream (LZ4_stream_t* streamPtr); </b><p> LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. LZ4_freeStream() releases its memory. - In the context of a DLL (liblz4), please use these methods rather than the static struct. - They are more future proof, in case of a change of `LZ4_stream_t` size. + +</p></pre><BR> + +<pre><b>void LZ4_resetStream (LZ4_stream_t* streamPtr); +</b><p> An LZ4_stream_t structure can be allocated once and re-used multiple times. + Use this function to start compressing a new stream. </p></pre><BR> <pre><b>int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); -</b><p> Use this function to load a static dictionary into LZ4_stream. +</b><p> Use this function to load a static dictionary into LZ4_stream_t. Any previous data will be forgotten, only 'dictionary' will remain in memory. - Loading a size of 0 is allowed. - Return : dictionary size, in bytes (necessarily <= 64 KB) + Loading a size of 0 is allowed, and is the same as reset. + @return : dictionary size, in bytes (necessarily <= 64 KB) </p></pre><BR> -<pre><b>int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); -</b><p> Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. - Important : Previous data blocks are assumed to still be present and unmodified ! +<pre><b>int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); +</b><p> Compress content into 'src' using data from previously compressed blocks, improving compression ratio. 'dst' buffer must be already allocated. - If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. + If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + + Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory ! + Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB. + Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + + @return : size of compressed block + or 0 if there is an error (typically, compressed data cannot fit into 'dst') + After an error, the stream status is invalid, it can only be reset or freed. </p></pre><BR> <pre><b>int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); -</b><p> If previously compressed data block is not guaranteed to remain available at its memory location, +</b><p> If previously compressed data block is not guaranteed to remain available at its current memory location, save it into a safer place (char* safeBuffer). - Note : you don't need to call LZ4_loadDict() afterwards, - dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable. + @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. </p></pre><BR> -<a name="Chapter7"></a><h2>Streaming Decompression Functions</h2><pre></pre> +<a name="Chapter7"></a><h2>Streaming Decompression Functions</h2><pre> Bufferless synchronous API +<BR></pre> -<pre><b>typedef struct { - union { - unsigned long long table[LZ4_STREAMDECODESIZE_U64]; - LZ4_streamDecode_t_internal internal_donotuse; - }; -</b></pre><BR> <pre><b>LZ4_streamDecode_t* LZ4_createStreamDecode(void); int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); -</b><p> information structure to track an LZ4 stream. - init this structure content using LZ4_setStreamDecode or memset() before first use ! +</b><p> creation / destruction of streaming decompression tracking structure. + A tracking structure can be re-used multiple times sequentially. +</p></pre><BR> + +<pre><b>int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); +</b><p> An LZ4_streamDecode_t structure can be allocated once and re-used multiple times. + Use this function to start decompression of a new stream of blocks. + A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order. + @return : 1 if OK, 0 if error + +</p></pre><BR> + +<pre><b>int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); +int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); +</b><p> These decoding functions allow decompression of consecutive blocks in "streaming" mode. + A block is an unsplittable entity, it must be presented entirely to a decompression function. + Decompression functions only accept one block at a time. + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB). - In the context of a DLL (liblz4) please prefer usage of construction methods below. - They are more future proof, in case of a change of LZ4_streamDecode_t size in the future. - LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure - LZ4_freeStreamDecode releases its memory. + Special : if application sets a ring buffer for decompression, it must respect one of the following conditions : + - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + maxBlockSize is implementation dependent. It's the maximum size of any single block. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including small ones ( < 64 KB). + - _At least_ 64 KB + 8 bytes + maxBlockSize. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including larger than decoding buffer. + Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block. +</p></pre><BR> + +<pre><b>int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); +int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); +</b><p> These decoding functions work the same as + a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() + They are stand-alone, and don't need an LZ4_streamDecode_t structure. </p></pre><BR> -<pre><b>int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); -</b><p> Use this function to instruct where to find the dictionary. - Setting a size of 0 is allowed (same effect as reset). - @return : 1 if OK, 0 if error +<a name="Chapter8"></a><h2>Private definitions</h2><pre> + Do not use these definitions. + They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + Using these definitions will expose code to API and/or ABI break in future versions of the library. +<BR></pre> + +<pre><b>typedef struct { + uint32_t hashTable[LZ4_HASH_SIZE_U32]; + uint32_t currentOffset; + uint32_t initCheck; + const uint8_t* dictionary; + uint8_t* bufferStart; </b>/* obsolete, used for slideInputBuffer */<b> + uint32_t dictSize; +} LZ4_stream_t_internal; +</b></pre><BR> +<pre><b>typedef struct { + const uint8_t* externalDict; + size_t extDictSize; + const uint8_t* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; +</b></pre><BR> +<pre><b>typedef struct { + unsigned int hashTable[LZ4_HASH_SIZE_U32]; + unsigned int currentOffset; + unsigned int initCheck; + const unsigned char* dictionary; + unsigned char* bufferStart; </b>/* obsolete, used for slideInputBuffer */<b> + unsigned int dictSize; +} LZ4_stream_t_internal; +</b></pre><BR> +<pre><b>typedef struct { + const unsigned char* externalDict; + size_t extDictSize; + const unsigned char* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; +</b></pre><BR> +<pre><b>#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) +union LZ4_stream_u { + unsigned long long table[LZ4_STREAMSIZE_U64]; + LZ4_stream_t_internal internal_donotuse; +} ; </b>/* previously typedef'd to LZ4_stream_t */<b> +</b><p> information structure to track an LZ4 stream. + init this structure before first use. + note : only use in association with static linking ! + this definition is not API/ABI safe, + it may change in a future version ! </p></pre><BR> -<pre><b>int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); -</b><p> These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - In the case of a ring buffers, decoding buffer must be either : - - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including small ones ( < 64 KB). - - _At least_ 64 KB + 8 bytes + maxBlockSize. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including larger than decoding buffer. - Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - and indicate where it is saved using LZ4_setStreamDecode() +<pre><b>#define LZ4_STREAMDECODESIZE_U64 4 +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +union LZ4_streamDecode_u { + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + LZ4_streamDecode_t_internal internal_donotuse; +} ; </b>/* previously typedef'd to LZ4_streamDecode_t */<b> +</b><p> information structure to track an LZ4 stream during decompression. + init this structure using LZ4_setStreamDecode (or memset()) before first use + note : only use in association with static linking ! + this definition is not API/ABI safe, + and may change in a future version ! + </p></pre><BR> -<pre><b>int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); -</b><p>Advanced decoding functions : - These decoding functions work the same as - a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue() - They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure. +<a name="Chapter9"></a><h2>Obsolete Functions</h2><pre></pre> + +<pre><b>#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS +# define LZ4_DEPRECATED(message) </b>/* disable deprecation warnings */<b> +#else +# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if defined(__clang__) </b>/* clang doesn't handle mixed C++11 and CNU attributes */<b> +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined (__cplusplus) && (__cplusplus >= 201402) </b>/* C++14 or greater */<b> +# define LZ4_DEPRECATED(message) [[deprecated(message)]] +# elif (LZ4_GCC_VERSION >= 405) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (LZ4_GCC_VERSION >= 301) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") +# define LZ4_DEPRECATED(message) +# endif +#endif </b>/* LZ4_DISABLE_DEPRECATE_WARNINGS */<b> +</b><p> Should deprecation warnings be a problem, + it is generally possible to disable them, + typically with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS </p></pre><BR> </html> diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html new file mode 100644 index 0000000..590c632 --- /dev/null +++ b/doc/lz4frame_manual.html @@ -0,0 +1,282 @@ +<html> +<head> +<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"> +<title>1.8.1 Manual</title> +</head> +<body> +<h1>1.8.1 Manual</h1> +<hr> +<a name="Contents"></a><h2>Contents</h2> +<ol> +<li><a href="#Chapter1">Introduction</a></li> +<li><a href="#Chapter2">Compiler specifics</a></li> +<li><a href="#Chapter3">Error management</a></li> +<li><a href="#Chapter4">Frame compression types</a></li> +<li><a href="#Chapter5">Simple compression function</a></li> +<li><a href="#Chapter6">Advanced compression functions</a></li> +<li><a href="#Chapter7">Resource Management</a></li> +<li><a href="#Chapter8">Compression</a></li> +<li><a href="#Chapter9">Decompression functions</a></li> +<li><a href="#Chapter10">Streaming decompression functions</a></li> +</ol> +<hr> +<a name="Chapter1"></a><h2>Introduction</h2><pre> + lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md). + lz4frame.h provides frame compression functions that take care + of encoding standard metadata alongside LZ4-compressed blocks. +<BR></pre> + +<a name="Chapter2"></a><h2>Compiler specifics</h2><pre></pre> + +<a name="Chapter3"></a><h2>Error management</h2><pre></pre> + +<pre><b>unsigned LZ4F_isError(LZ4F_errorCode_t code); </b>/**< tells if a `LZ4F_errorCode_t` function result is an error code */<b> +</b></pre><BR> +<pre><b>const char* LZ4F_getErrorName(LZ4F_errorCode_t code); </b>/**< return error code string; useful for debugging */<b> +</b></pre><BR> +<a name="Chapter4"></a><h2>Frame compression types</h2><pre></pre> + +<pre><b>typedef enum { + LZ4F_default=0, + LZ4F_max64KB=4, + LZ4F_max256KB=5, + LZ4F_max1MB=6, + LZ4F_max4MB=7 + LZ4F_OBSOLETE_ENUM(max64KB) + LZ4F_OBSOLETE_ENUM(max256KB) + LZ4F_OBSOLETE_ENUM(max1MB) + LZ4F_OBSOLETE_ENUM(max4MB) +} LZ4F_blockSizeID_t; +</b></pre><BR> +<pre><b>typedef enum { + LZ4F_blockLinked=0, + LZ4F_blockIndependent + LZ4F_OBSOLETE_ENUM(blockLinked) + LZ4F_OBSOLETE_ENUM(blockIndependent) +} LZ4F_blockMode_t; +</b></pre><BR> +<pre><b>typedef enum { + LZ4F_noContentChecksum=0, + LZ4F_contentChecksumEnabled + LZ4F_OBSOLETE_ENUM(noContentChecksum) + LZ4F_OBSOLETE_ENUM(contentChecksumEnabled) +} LZ4F_contentChecksum_t; +</b></pre><BR> +<pre><b>typedef enum { + LZ4F_noBlockChecksum=0, + LZ4F_blockChecksumEnabled +} LZ4F_blockChecksum_t; +</b></pre><BR> +<pre><b>typedef enum { + LZ4F_frame=0, + LZ4F_skippableFrame + LZ4F_OBSOLETE_ENUM(skippableFrame) +} LZ4F_frameType_t; +</b></pre><BR> +<pre><b>typedef struct { + LZ4F_blockSizeID_t blockSizeID; </b>/* max64KB, max256KB, max1MB, max4MB ; 0 == default */<b> + LZ4F_blockMode_t blockMode; </b>/* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */<b> + LZ4F_contentChecksum_t contentChecksumFlag; </b>/* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */<b> + LZ4F_frameType_t frameType; </b>/* read-only field : LZ4F_frame or LZ4F_skippableFrame */<b> + unsigned long long contentSize; </b>/* Size of uncompressed content ; 0 == unknown */<b> + unsigned dictID; </b>/* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */<b> + LZ4F_blockChecksum_t blockChecksumFlag; </b>/* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */<b> +} LZ4F_frameInfo_t; +</b><p> makes it possible to set or read frame parameters. + It's not required to set all fields, as long as the structure was initially memset() to zero. + For all fields, 0 sets it to default value +</p></pre><BR> + +<pre><b>typedef struct { + LZ4F_frameInfo_t frameInfo; + int compressionLevel; </b>/* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */<b> + unsigned autoFlush; </b>/* 1 == always flush, to reduce usage of internal buffers */<b> + unsigned reserved[4]; </b>/* must be zero for forward compatibility */<b> +} LZ4F_preferences_t; +</b><p> makes it possible to supply detailed compression parameters to the stream interface. + It's not required to set all fields, as long as the structure was initially memset() to zero. + All reserved fields must be set to zero. +</p></pre><BR> + +<a name="Chapter5"></a><h2>Simple compression function</h2><pre></pre> + +<pre><b>size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); +</b><p> Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. + Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. + +</p></pre><BR> + +<pre><b>size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr); +</b><p> Compress an entire srcBuffer into a valid LZ4 frame. + dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + @return : number of bytes written into dstBuffer. + or an error code if it fails (can be tested using LZ4F_isError()) + +</p></pre><BR> + +<a name="Chapter6"></a><h2>Advanced compression functions</h2><pre></pre> + +<pre><b>typedef struct { + unsigned stableSrc; </b>/* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */<b> + unsigned reserved[3]; +} LZ4F_compressOptions_t; +</b></pre><BR> +<a name="Chapter7"></a><h2>Resource Management</h2><pre></pre> + +<pre><b>LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); +LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); +</b><p> The first thing to do is to create a compressionContext object, which will be used in all compression operations. + This is achieved using LZ4F_createCompressionContext(), which takes as argument a version. + The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. + The function will provide a pointer to a fully allocated LZ4F_cctx object. + If @return != zero, there was an error during context creation. + Object can release its memory using LZ4F_freeCompressionContext(); + +</p></pre><BR> + +<a name="Chapter8"></a><h2>Compression</h2><pre></pre> + +<pre><b>size_t LZ4F_compressBegin(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* prefsPtr); +</b><p> will write the frame header into dstBuffer. + dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. + @return : number of bytes written into dstBuffer for the header + or an error code (which can be tested using LZ4F_isError()) + +</p></pre><BR> + +<pre><b>size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); +</b><p> Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + +</p></pre><BR> + +<pre><b>size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); +</b><p> LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. + This value is provided by LZ4F_compressBound(). + If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). + LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. + `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. + @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). + or an error code if it fails (which can be tested using LZ4F_isError()) + +</p></pre><BR> + +<pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); +</b><p> When data must be generated and sent immediately, without waiting for a block to be completely filled, + it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. + `dstCapacity` must be large enough to ensure the operation will be successful. + `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx) + or an error code if it fails (which can be tested using LZ4F_isError()) + +</p></pre><BR> + +<pre><b>size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); +</b><p> To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). + It will flush whatever data remained within `cctx` (like LZ4_flush()) + and properly finalize the frame, with an endMark and a checksum. + `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. + @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) + or an error code if it fails (which can be tested using LZ4F_isError()) + A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. + +</p></pre><BR> + +<a name="Chapter9"></a><h2>Decompression functions</h2><pre></pre> + +<pre><b>typedef struct { + unsigned stableDst; </b>/* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */<b> + unsigned reserved[3]; </b>/* must be set to zero for forward compatibility */<b> +} LZ4F_decompressOptions_t; +</b></pre><BR> +<pre><b>LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version); +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); +</b><p> Create an LZ4F_dctx object, to track all decompression operations. + The version provided MUST be LZ4F_VERSION. + The function provides a pointer to an allocated and initialized LZ4F_dctx object. + The result is an errorCode, which can be tested using LZ4F_isError(). + dctx memory can be released using LZ4F_freeDecompressionContext(); + The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. + That is, it should be == 0 if decompression has been completed fully and correctly. + +</p></pre><BR> + +<a name="Chapter10"></a><h2>Streaming decompression functions</h2><pre></pre> + +<pre><b>size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr); +</b><p> This function extracts frame parameters (max blockSize, dictID, etc.). + Its usage is optional. + Extracted information is typically useful for allocation and dictionary. + This function works in 2 situations : + - At the beginning of a new frame, in which case + it will decode information from `srcBuffer`, starting the decoding process. + Input size must be large enough to successfully decode the entire frame header. + Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. + It's allowed to provide more input data than this minimum. + - After decoding has been started. + In which case, no input is read, frame parameters are extracted from dctx. + - If decoding has barely started, but not yet extracted information from header, + LZ4F_getFrameInfo() will fail. + The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + Decompression must resume from (srcBuffer + *srcSizePtr). + @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, + or an error code which can be tested using LZ4F_isError(). + note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. + note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. + +</p></pre><BR> + +<pre><b>size_t LZ4F_decompress(LZ4F_dctx* dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* dOptPtr); +</b><p> Call this function repetitively to regenerate compressed data from `srcBuffer`. + The function will read up to *srcSizePtr bytes from srcBuffer, + and decompress data into dstBuffer, of capacity *dstSizePtr. + + The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value). + The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value). + + The function does not necessarily read all input bytes, so always check value in *srcSizePtr. + Unconsumed source data must be presented again in subsequent invocations. + + `dstBuffer` can freely change between each consecutive function invocation. + `dstBuffer` content will be overwritten. + + @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. + Schematically, it's the size of the current (or remaining) compressed block + header of next block. + Respecting the hint provides some small speed benefit, because it skips intermediate buffers. + This is just a hint though, it's always possible to provide any srcSize. + + When a frame is fully decoded, @return will be 0 (no more data expected). + When provided with more bytes than necessary to decode a frame, + LZ4F_decompress() will stop reading exactly at end of current frame, and @return 0. + + If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). + After a decompression error, the `dctx` context is not resumable. + Use LZ4F_resetDecompressionContext() to return to clean state. + + After a frame is fully decoded, dctx can be used again to decompress another frame. + +</p></pre><BR> + +<pre><b>void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); </b>/* always successful */<b> +</b><p> In case of an error, the context is left in "undefined" state. + In which case, it's necessary to reset it, before re-using it. + This method can also be used to abruptly stop any unfinished decompression, + and start a new one using same context resources. +</p></pre><BR> + +</html> +</body> diff --git a/examples/.gitignore b/examples/.gitignore index 3ceb90d..5abeef6 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -6,4 +6,5 @@ /ringBufferHC /lineCompress /frameCompress +/simpleBuffer /*.exe diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c index d49b267..a878577 100644 --- a/examples/HCStreaming_ringBuffer.c +++ b/examples/HCStreaming_ringBuffer.c @@ -1,12 +1,12 @@ // LZ4 HC streaming API example : ring buffer -// Based on previous work from Takayuki Matsuoka +// Based on a previous example by Takayuki Matsuoka /************************************** * Compiler Options **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* for MSVC */ +#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ +# define _CRT_SECURE_NO_WARNINGS # define snprintf sprintf_s #endif diff --git a/examples/Makefile b/examples/Makefile index aad713b..9321c24 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -27,13 +27,14 @@ # kindly provided by Takayuki Matsuoka # ########################################################################## -CFLAGS ?= -O3 -CFLAGS += -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -FLAGS := -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CPPFLAGS += -I../lib +CFLAGS ?= -O3 +CFLAGS += -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes +FLAGS := $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) -TESTFILE= Makefile -LZ4DIR := ../lib -LZ4 = ../programs/lz4 +TESTFILE = Makefile +LZ4DIR := ../lib +LZ4 = ../programs/lz4 # Define *.exe as extension for Windows systems @@ -48,7 +49,8 @@ endif default: all -all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC lineCompress frameCompress +all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \ + lineCompress frameCompress simpleBuffer printVersion: $(LZ4DIR)/lz4.c printVersion.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -78,12 +80,21 @@ simpleBuffer: $(LZ4DIR)/lz4.c simple_buffer.c $(CC) $(FLAGS) $^ -o $@$(EXT) test : all + @echo "\n=== Print Version ===" ./printVersion$(EXT) + @echo "\n=== Simple compression example ===" + ./simpleBuffer$(EXT) + @echo "\n=== Double-buffer ===" ./doubleBuffer$(EXT) $(TESTFILE) - ./dictionaryRandomAccess$(EXT) $(TESTFILE) $(TESTFILE) 1100 1400 + @echo "\n=== Ring Buffer ===" ./ringBuffer$(EXT) $(TESTFILE) + @echo "\n=== Ring Buffer + LZ4 HC ===" ./ringBufferHC$(EXT) $(TESTFILE) + @echo "\n=== Compress line by line ===" ./lineCompress$(EXT) $(TESTFILE) + @echo "\n=== Dictionary Random Access ===" + ./dictionaryRandomAccess$(EXT) $(TESTFILE) $(TESTFILE) 1100 1400 + @echo "\n=== Frame compression ===" ./frameCompress$(EXT) $(TESTFILE) $(LZ4) -vt $(TESTFILE).lz4 diff --git a/examples/blockStreaming_doubleBuffer.c b/examples/blockStreaming_doubleBuffer.c index d02f258..acb3455 100644 --- a/examples/blockStreaming_doubleBuffer.c +++ b/examples/blockStreaming_doubleBuffer.c @@ -2,7 +2,7 @@ // Copyright : Takayuki Matsuoka -#ifdef _MSC_VER /* Visual Studio */ +#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ # define _CRT_SECURE_NO_WARNINGS # define snprintf sprintf_s #endif diff --git a/examples/blockStreaming_lineByLine.c b/examples/blockStreaming_lineByLine.c index f449aa3..677c426 100644 --- a/examples/blockStreaming_lineByLine.c +++ b/examples/blockStreaming_lineByLine.c @@ -1,8 +1,8 @@ // LZ4 streaming API example : line-by-line logfile compression -// Copyright : Takayuki Matsuoka +// by Takayuki Matsuoka -#ifdef _MSC_VER /* Visual Studio */ +#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ # define _CRT_SECURE_NO_WARNINGS # define snprintf sprintf_s #endif diff --git a/examples/blockStreaming_ringBuffer.c b/examples/blockStreaming_ringBuffer.c index 697d342..0b6a3ce 100644 --- a/examples/blockStreaming_ringBuffer.c +++ b/examples/blockStreaming_ringBuffer.c @@ -1,17 +1,14 @@ -// LZ4 streaming API example : ring buffer -// Based on sample code from Takayuki Matsuoka +/* LZ4 streaming API example : ring buffer + * Based on sample code from Takayuki Matsuoka */ /************************************** * Compiler Options **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS // for MSVC +#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ +# define _CRT_SECURE_NO_WARNINGS # define snprintf sprintf_s #endif -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif /************************************** @@ -27,7 +24,7 @@ enum { MESSAGE_MAX_BYTES = 1024, RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES, - DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger, to test unsynchronized ring buffers + DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES /* Intentionally larger, to test unsynchronized ring buffers */ }; @@ -50,7 +47,7 @@ size_t read_bin(FILE* fp, void* array, int arrayBytes) { void test_compress(FILE* outFp, FILE* inpFp) { - LZ4_stream_t lz4Stream_body = { 0 }; + LZ4_stream_t lz4Stream_body = { { 0 } }; LZ4_stream_t* lz4Stream = &lz4Stream_body; static char inpBuf[RING_BUFFER_BYTES]; @@ -85,24 +82,22 @@ void test_compress(FILE* outFp, FILE* inpFp) void test_decompress(FILE* outFp, FILE* inpFp) { static char decBuf[DECODE_RING_BUFFER]; - int decOffset = 0; - LZ4_streamDecode_t lz4StreamDecode_body = { 0 }; + int decOffset = 0; + LZ4_streamDecode_t lz4StreamDecode_body = { { 0 } }; LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; for(;;) { int cmpBytes = 0; char cmpBuf[CMPBUFSIZE]; - { - const size_t r0 = read_int32(inpFp, &cmpBytes); + { const size_t r0 = read_int32(inpFp, &cmpBytes); if(r0 != 1 || cmpBytes <= 0) break; const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes); if(r1 != (size_t) cmpBytes) break; } - { - char* const decPtr = &decBuf[decOffset]; + { char* const decPtr = &decBuf[decOffset]; const int decBytes = LZ4_decompress_safe_continue( lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); if(decBytes <= 0) break; @@ -120,7 +115,7 @@ int compare(FILE* f0, FILE* f1) { int result = 0; - while(0 == result) { + while (0 == result) { char b0[65536]; char b1[65536]; const size_t r0 = fread(b0, 1, sizeof(b0), f0); @@ -128,12 +123,9 @@ int compare(FILE* f0, FILE* f1) result = (int) r0 - (int) r1; - if(0 == r0 || 0 == r1) { - break; - } - if(0 == result) { - result = memcmp(b0, b1, r0); - } + if (0 == r0 || 0 == r1) break; + + if (0 == result) result = memcmp(b0, b1, r0); } return result; @@ -146,7 +138,7 @@ int main(int argc, char** argv) char lz4Filename[256] = { 0 }; char decFilename[256] = { 0 }; - if(argc < 2) { + if (argc < 2) { printf("Please specify input filename\n"); return 0; } @@ -160,9 +152,8 @@ int main(int argc, char** argv) printf("dec = [%s]\n", decFilename); // compress - { - FILE* inpFp = fopen(inpFilename, "rb"); - FILE* outFp = fopen(lz4Filename, "wb"); + { FILE* const inpFp = fopen(inpFilename, "rb"); + FILE* const outFp = fopen(lz4Filename, "wb"); test_compress(outFp, inpFp); @@ -171,9 +162,8 @@ int main(int argc, char** argv) } // decompress - { - FILE* inpFp = fopen(lz4Filename, "rb"); - FILE* outFp = fopen(decFilename, "wb"); + { FILE* const inpFp = fopen(lz4Filename, "rb"); + FILE* const outFp = fopen(decFilename, "wb"); test_decompress(outFp, inpFp); @@ -182,12 +172,11 @@ int main(int argc, char** argv) } // verify - { - FILE* inpFp = fopen(inpFilename, "rb"); - FILE* decFp = fopen(decFilename, "rb"); + { FILE* const inpFp = fopen(inpFilename, "rb"); + FILE* const decFp = fopen(decFilename, "rb"); const int cmp = compare(inpFp, decFp); - if(0 == cmp) { + if (0 == cmp) { printf("Verify : OK\n"); } else { printf("Verify : NG\n"); diff --git a/examples/dictionaryRandomAccess.c b/examples/dictionaryRandomAccess.c index 6acf99b..291fd08 100644 --- a/examples/dictionaryRandomAccess.c +++ b/examples/dictionaryRandomAccess.c @@ -1,6 +1,6 @@ // LZ4 API example : Dictionary Random Access -#ifdef _MSC_VER /* Visual Studio */ +#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ # define _CRT_SECURE_NO_WARNINGS # define snprintf sprintf_s #endif diff --git a/examples/frameCompress.c b/examples/frameCompress.c index 75f1576..d66a8dc 100644 --- a/examples/frameCompress.c +++ b/examples/frameCompress.c @@ -13,299 +13,303 @@ #define LZ4_FOOTER_SIZE 4 static const LZ4F_preferences_t lz4_preferences = { - { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0, { 0, 0 } }, - 0, /* compression level */ - 0, /* autoflush */ - { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ + { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, + 0 /* content size unknown */, 0 /* no dictID */ , LZ4F_noBlockChecksum }, + 0, /* compression level */ + 0, /* autoflush */ + { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ }; static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) { - LZ4F_errorCode_t r; - LZ4F_compressionContext_t ctx; - char *src, *buf = NULL; - size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size; - - r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(r)) { - printf("Failed to create context: error %zu\n", r); - return 1; - } - r = 1; - - src = malloc(BUF_SIZE); - if (!src) { - printf("Not enough memory\n"); - goto cleanup; - } - - frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences); - size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; - buf = malloc(size); - if (!buf) { - printf("Not enough memory\n"); - goto cleanup; - } - - n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences); - if (LZ4F_isError(n)) { - printf("Failed to start compression: error %zu\n", n); - goto cleanup; - } - - printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n); - - for (;;) { - k = fread(src, 1, BUF_SIZE, in); - if (k == 0) - break; - count_in += k; - - n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL); - if (LZ4F_isError(n)) { - printf("Compression failed: error %zu\n", n); - goto cleanup; - } - - offset += n; - count_out += n; - if (size - offset < frame_size + LZ4_FOOTER_SIZE) { - printf("Writing %zu bytes\n", offset); - - k = fwrite(buf, 1, offset, out); - if (k < offset) { - if (ferror(out)) - printf("Write failed\n"); - else - printf("Short write\n"); - goto cleanup; - } - - offset = 0; - } - } - - n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL); - if (LZ4F_isError(n)) { - printf("Failed to end compression: error %zu\n", n); - goto cleanup; - } - - offset += n; - count_out += n; - printf("Writing %zu bytes\n", offset); - - k = fwrite(buf, 1, offset, out); - if (k < offset) { - if (ferror(out)) - printf("Write failed\n"); - else - printf("Short write\n"); - goto cleanup; - } - - *size_in = count_in; - *size_out = count_out; - r = 0; + size_t r=1; /* function result; 1 == error, default (early exit) */ + LZ4F_compressionContext_t ctx; + char *src, *buf = NULL; + size_t size, count_in = 0, count_out, offset = 0, frame_size; + + if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) { + printf("Failed to create context: error %zu\n", r); + return 1; + } + + src = malloc(BUF_SIZE); + if (!src) { + printf("Not enough memory\n"); + goto cleanup; + } + + frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences); + size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; + buf = malloc(size); + if (!buf) { + printf("Not enough memory\n"); + goto cleanup; + } + + { size_t const headerSize = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences); + if (LZ4F_isError(headerSize)) { + printf("Failed to start compression: error %zu\n", headerSize); + goto cleanup; + } + offset = count_out = headerSize; + printf("Buffer size is %zu bytes, header size %zu bytes\n", size, headerSize); + } + + + for (;;) { + size_t const readSize = fread(src, 1, BUF_SIZE, in); + if (readSize == 0) + break; + count_in += readSize; + + { size_t const compressedSize = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, readSize, NULL); + if (LZ4F_isError(compressedSize)) { + printf("Compression failed: error %zu\n", compressedSize); + goto cleanup; + } + offset += compressedSize; + count_out += compressedSize; + } + + if (size - offset < frame_size + LZ4_FOOTER_SIZE) { + size_t writtenSize; + printf("Writing %zu bytes\n", offset); + + writtenSize = fwrite(buf, 1, offset, out); + if (writtenSize < offset) { + if (ferror(out)) /* note : ferror() must follow fwrite */ + printf("Write failed\n"); + else + printf("Short write\n"); + goto cleanup; + } + + offset = 0; + } + } + + { size_t const compressedSize = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL); + if (LZ4F_isError(compressedSize)) { + printf("Failed to end compression: error %zu\n", compressedSize); + goto cleanup; + } + offset += compressedSize; + count_out += compressedSize; + } + + printf("Writing %zu bytes\n", offset); + { size_t const writtenSize = fwrite(buf, 1, offset, out); + if (writtenSize < offset) { + if (ferror(out)) + printf("Write failed\n"); + else + printf("Short write\n"); + goto cleanup; + } } + + *size_in = count_in; + *size_out = count_out; + r = 0; /* success */ + cleanup: - if (ctx) - LZ4F_freeCompressionContext(ctx); - free(src); - free(buf); - return r; + LZ4F_freeCompressionContext(ctx); /* supports free on NULL */ + free(src); + free(buf); + return r; } static size_t get_block_size(const LZ4F_frameInfo_t* info) { - switch (info->blockSizeID) { + switch (info->blockSizeID) { case LZ4F_default: - case LZ4F_max64KB: return 1 << 16; - case LZ4F_max256KB: return 1 << 18; - case LZ4F_max1MB: return 1 << 20; - case LZ4F_max4MB: return 1 << 22; - default: - printf("Impossible unless more block sizes are allowed\n"); - exit(1); - } + case LZ4F_max64KB: return 1 << 16; + case LZ4F_max256KB: return 1 << 18; + case LZ4F_max1MB: return 1 << 20; + case LZ4F_max4MB: return 1 << 22; + default: + printf("Impossible unless more block sizes are allowed\n"); + exit(1); + } } -static size_t decompress_file(FILE *in, FILE *out) { - void* const src = malloc(BUF_SIZE); - void* dst = NULL; - size_t dstCapacity = 0; - LZ4F_dctx *dctx = NULL; - size_t ret; +static size_t decompress_file(FILE* in, FILE* out) { + void* const src = malloc(BUF_SIZE); + void* dst = NULL; + size_t dstCapacity = 0; + LZ4F_dctx* dctx = NULL; + size_t ret = 1; - /* Initialization */ + /* Initialization */ if (!src) { perror("decompress_file(src)"); goto cleanup; } - ret = LZ4F_createDecompressionContext(&dctx, 100); - if (LZ4F_isError(ret)) { - printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret)); - goto cleanup; - } - - /* Decompression */ - ret = 1; - while (ret != 0) { - /* Load more input */ - size_t srcSize = fread(src, 1, BUF_SIZE, in); - void* srcPtr = src; - void* srcEnd = srcPtr + srcSize; - if (srcSize == 0 || ferror(in)) { - printf("Decompress: not enough input or error reading file\n"); - goto cleanup; - } - /* Allocate destination buffer if it isn't already */ - if (!dst) { - LZ4F_frameInfo_t info; - ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize); - if (LZ4F_isError(ret)) { - printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret)); - goto cleanup; - } - /* Allocating enough space for an entire block isn't necessary for - * correctness, but it allows some memcpy's to be elided. - */ - dstCapacity = get_block_size(&info); - dst = malloc(dstCapacity); + { size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100); + if (LZ4F_isError(dctxStatus)) { + printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus)); + goto cleanup; + } } + + /* Decompression */ + while (ret != 0) { + /* Load more input */ + size_t srcSize = fread(src, 1, BUF_SIZE, in); + const void* srcPtr = src; + const void* const srcEnd = srcPtr + srcSize; + if (srcSize == 0 || ferror(in)) { + printf("Decompress: not enough input or error reading file\n"); + goto cleanup; + } + /* Allocate destination buffer if it isn't already */ + if (!dst) { + LZ4F_frameInfo_t info; + ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize); + if (LZ4F_isError(ret)) { + printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret)); + goto cleanup; + } + /* Allocating enough space for an entire block isn't necessary for + * correctness, but it allows some memcpy's to be elided. + */ + dstCapacity = get_block_size(&info); + dst = malloc(dstCapacity); if (!dst) { perror("decompress_file(dst)"); goto cleanup; } - srcPtr += srcSize; - srcSize = srcEnd - srcPtr; - } - /* Decompress: - * Continue while there is more input to read and the frame isn't over. - * If srcPtr == srcEnd then we know that there is no more output left in the - * internal buffer left to flush. - */ - while (srcPtr != srcEnd && ret != 0) { - /* INVARIANT: Any data left in dst has already been written */ - size_t dstSize = dstCapacity; - ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL); - if (LZ4F_isError(ret)) { - printf("Decompression error: %s\n", LZ4F_getErrorName(ret)); - goto cleanup; - } - /* Flush output */ - if (dstSize != 0){ - size_t written = fwrite(dst, 1, dstSize, out); - printf("Writing %zu bytes\n", dstSize); - if (written != dstSize) { - printf("Decompress: Failed to write to file\n"); - goto cleanup; - } - } - /* Update input */ - srcPtr += srcSize; - srcSize = srcEnd - srcPtr; - } - } - /* Check that there isn't trailing input data after the frame. - * It is valid to have multiple frames in the same file, but this example - * doesn't support it. - */ - ret = fread(src, 1, 1, in); - if (ret != 0 || !feof(in)) { - printf("Decompress: Trailing data left in file after frame\n"); - goto cleanup; - } + srcPtr += srcSize; + srcSize = srcEnd - srcPtr; + } + /* Decompress: + * Continue while there is more input to read and the frame isn't over. + * If srcPtr == srcEnd then we know that there is no more output left in the + * internal buffer left to flush. + */ + while (srcPtr != srcEnd && ret != 0) { + /* INVARIANT: Any data left in dst has already been written */ + size_t dstSize = dstCapacity; + ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL); + if (LZ4F_isError(ret)) { + printf("Decompression error: %s\n", LZ4F_getErrorName(ret)); + goto cleanup; + } + /* Flush output */ + if (dstSize != 0){ + size_t written = fwrite(dst, 1, dstSize, out); + printf("Writing %zu bytes\n", dstSize); + if (written != dstSize) { + printf("Decompress: Failed to write to file\n"); + goto cleanup; + } + } + /* Update input */ + srcPtr += srcSize; + srcSize = srcEnd - srcPtr; + } + } + /* Check that there isn't trailing input data after the frame. + * It is valid to have multiple frames in the same file, but this example + * doesn't support it. + */ + ret = fread(src, 1, 1, in); + if (ret != 0 || !feof(in)) { + printf("Decompress: Trailing data left in file after frame\n"); + goto cleanup; + } cleanup: - free(src); - free(dst); - return LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */ + free(src); + free(dst); + return LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */ } int compare(FILE* fp0, FILE* fp1) { - int result = 0; + int result = 0; - while(0 == result) { - char b0[1024]; - char b1[1024]; - const size_t r0 = fread(b0, 1, sizeof(b0), fp0); - const size_t r1 = fread(b1, 1, sizeof(b1), fp1); + while(0 == result) { + char b0[1024]; + char b1[1024]; + const size_t r0 = fread(b0, 1, sizeof(b0), fp0); + const size_t r1 = fread(b1, 1, sizeof(b1), fp1); - result = (int) r0 - (int) r1; + result = (int) r0 - (int) r1; - if (0 == r0 || 0 == r1) { - break; - } - if (0 == result) { - result = memcmp(b0, b1, r0); - } - } + if (0 == r0 || 0 == r1) { + break; + } + if (0 == result) { + result = memcmp(b0, b1, r0); + } + } - return result; + return result; } int main(int argc, const char **argv) { - char inpFilename[256] = { 0 }; - char lz4Filename[256] = { 0 }; - char decFilename[256] = { 0 }; - - if(argc < 2) { - printf("Please specify input filename\n"); - return 0; - } - - snprintf(inpFilename, 256, "%s", argv[1]); - snprintf(lz4Filename, 256, "%s.lz4", argv[1]); - snprintf(decFilename, 256, "%s.lz4.dec", argv[1]); - - printf("inp = [%s]\n", inpFilename); - printf("lz4 = [%s]\n", lz4Filename); - printf("dec = [%s]\n", decFilename); - - /* compress */ - { FILE* const inpFp = fopen(inpFilename, "rb"); - FILE* const outFp = fopen(lz4Filename, "wb"); - size_t sizeIn = 0; - size_t sizeOut = 0; - size_t ret; - - printf("compress : %s -> %s\n", inpFilename, lz4Filename); - ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut); - if (ret) { - printf("compress : failed with code %zu\n", ret); - return ret; - } - printf("%s: %zu → %zu bytes, %.1f%%\n", - inpFilename, sizeIn, sizeOut, - (double)sizeOut / sizeIn * 100); - printf("compress : done\n"); - - fclose(outFp); - fclose(inpFp); - } - - /* decompress */ - { FILE* const inpFp = fopen(lz4Filename, "rb"); - FILE* const outFp = fopen(decFilename, "wb"); - size_t ret; - - printf("decompress : %s -> %s\n", lz4Filename, decFilename); - ret = decompress_file(inpFp, outFp); - if (ret) { - printf("decompress : failed with code %zu\n", ret); - return ret; - } - printf("decompress : done\n"); - - fclose(outFp); - fclose(inpFp); - } - - /* verify */ - { FILE* const inpFp = fopen(inpFilename, "rb"); - FILE* const decFp = fopen(decFilename, "rb"); - - printf("verify : %s <-> %s\n", inpFilename, decFilename); - const int cmp = compare(inpFp, decFp); - if(0 == cmp) { - printf("verify : OK\n"); - } else { - printf("verify : NG\n"); - } - - fclose(decFp); - fclose(inpFp); - } + char inpFilename[256] = { 0 }; + char lz4Filename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if(argc < 2) { + printf("Please specify input filename\n"); + return 0; + } + + snprintf(inpFilename, 256, "%s", argv[1]); + snprintf(lz4Filename, 256, "%s.lz4", argv[1]); + snprintf(decFilename, 256, "%s.lz4.dec", argv[1]); + + printf("inp = [%s]\n", inpFilename); + printf("lz4 = [%s]\n", lz4Filename); + printf("dec = [%s]\n", decFilename); + + /* compress */ + { FILE* const inpFp = fopen(inpFilename, "rb"); + FILE* const outFp = fopen(lz4Filename, "wb"); + size_t sizeIn = 0; + size_t sizeOut = 0; + size_t ret; + + printf("compress : %s -> %s\n", inpFilename, lz4Filename); + ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut); + if (ret) { + printf("compress : failed with code %zu\n", ret); + return (int)ret; + } + printf("%s: %zu → %zu bytes, %.1f%%\n", + inpFilename, sizeIn, sizeOut, + (double)sizeOut / sizeIn * 100); + printf("compress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + /* decompress */ + { FILE* const inpFp = fopen(lz4Filename, "rb"); + FILE* const outFp = fopen(decFilename, "wb"); + size_t ret; + + printf("decompress : %s -> %s\n", lz4Filename, decFilename); + ret = decompress_file(inpFp, outFp); + if (ret) { + printf("decompress : failed with code %zu\n", ret); + return (int)ret; + } + printf("decompress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + /* verify */ + { FILE* const inpFp = fopen(inpFilename, "rb"); + FILE* const decFp = fopen(decFilename, "rb"); + + printf("verify : %s <-> %s\n", inpFilename, decFilename); + const int cmp = compare(inpFp, decFp); + if(0 == cmp) { + printf("verify : OK\n"); + } else { + printf("verify : NG\n"); + } + + fclose(decFp); + fclose(inpFp); + } + + return 0; } diff --git a/examples/printVersion.c b/examples/printVersion.c index 8607139..7af318a 100644 --- a/examples/printVersion.c +++ b/examples/printVersion.c @@ -1,5 +1,5 @@ // LZ4 trivial example : print Library version number -// Copyright : Takayuki Matsuoka & Yann Collet +// by Takayuki Matsuoka #include <stdio.h> diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c index f531d48..403d9e8 100644 --- a/examples/simple_buffer.c +++ b/examples/simple_buffer.c @@ -8,10 +8,10 @@ */ /* Includes, for Power! */ -#include "lz4.h" // This is all that is required to expose the prototypes for basic compression and decompression. -#include <stdio.h> // For printf() -#include <string.h> // For memcmp() -#include <stdlib.h> // For exit() +#include "lz4.h" // This is all that is required to expose the prototypes for basic compression and decompression. +#include <stdio.h> // For printf() +#include <string.h> // For memcmp() +#include <stdlib.h> // For exit() /* * Easy show-error-and-bail function. @@ -28,37 +28,39 @@ void run_screaming(const char *message, const int code) { */ int main(void) { /* Introduction */ - // Below we will have a Compression and Decompression section to demonstrate. There are a few important notes before we start: - // 1) The return codes of LZ4_ functions are important. Read lz4.h if you're unsure what a given code means. - // 2) LZ4 uses char* pointers in all LZ4_ functions. This is baked into the API and probably not going to change. If your - // program uses pointers that are unsigned char*, void*, or otherwise different you may need to do some casting or set the - // right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign). + // Below we will have a Compression and Decompression section to demonstrate. + // There are a few important notes before we start: + // 1) The return codes of LZ4_ functions are important. + // Read lz4.h if you're unsure what a given code means. + // 2) LZ4 uses char* pointers in all LZ4_ functions. + // This is baked into the API and probably not going to change. + // If your program uses pointers that are unsigned char*, void*, or otherwise different, + // you may need to do some casting or set the right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign). /* Compression */ // We'll store some text into a variable pointed to by *src to be compressed later. - const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; - // The compression function needs to know how many bytes of exist. Since we're using a string, we can use strlen() + 1 (for \0). - const size_t src_size = strlen(src) + 1; + const char* const src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; + // The compression function needs to know how many bytes exist. Since we're using a string, we can use strlen() + 1 (for \0). + const int src_size = (int)(strlen(src) + 1); // LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound(). - const size_t max_dst_size = LZ4_compressBound(src_size); + const int max_dst_size = LZ4_compressBound(src_size); // We will use that size for our destination boundary when allocating space. - char *compressed_data = malloc(max_dst_size); + char* compressed_data = malloc(max_dst_size); if (compressed_data == NULL) run_screaming("Failed to allocate memory for *compressed_data.", 1); - // That's all the information and preparation LZ4 needs to compress *src into *compressed_data. Invoke LZ4_compress_default now - // with our size values and pointers to our memory locations. Save the return value for error checking. - int return_value = 0; - return_value = LZ4_compress_default(src, compressed_data, src_size, max_dst_size); + // That's all the information and preparation LZ4 needs to compress *src into *compressed_data. + // Invoke LZ4_compress_default now with our size values and pointers to our memory locations. + // Save the return value for error checking. + const int compressed_data_size = LZ4_compress_default(src, compressed_data, src_size, max_dst_size); // Check return_value to determine what happened. - if (return_value < 0) - run_screaming("A negative result from LZ4_compress_default indicates a failure trying to compress the data. See exit code (echo $?) for value returned.", return_value); - if (return_value == 0) + if (compressed_data_size < 0) + run_screaming("A negative result from LZ4_compress_default indicates a failure trying to compress the data. See exit code (echo $?) for value returned.", compressed_data_size); + if (compressed_data_size == 0) run_screaming("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.", 1); - if (return_value > 0) + if (compressed_data_size > 0) printf("We successfully compressed some data!\n"); - // Not only does a positive return_value mean success, the value returned == the number of bytes required. You can use this to - // realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept. - const size_t compressed_data_size = return_value; + // Not only does a positive return_value mean success, the value returned == the number of bytes required. + // You can use this to realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept. compressed_data = (char *)realloc(compressed_data, compressed_data_size); if (compressed_data == NULL) run_screaming("Failed to re-alloc memory for compressed_data. Sad :(", 1); @@ -66,25 +68,27 @@ int main(void) { /* Decompression */ // Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite! We'll create a // *new_src location of size src_size since we know that value. - char *new_src = malloc(src_size); - if (new_src == NULL) - run_screaming("Failed to allocate memory for *new_src.", 1); - // The LZ4_decompress_safe function needs to know where the compressed data is, how many bytes long it is, where the new_src - // memory location is, and how large the new_src (uncompressed) output will be. Again, save the return_value. - return_value = LZ4_decompress_safe(compressed_data, new_src, compressed_data_size, src_size); - if (return_value < 0) - run_screaming("A negative result from LZ4_decompress_fast indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", return_value); - if (return_value == 0) + char* const regen_buffer = malloc(src_size); + if (regen_buffer == NULL) + run_screaming("Failed to allocate memory for *regen_buffer.", 1); + // The LZ4_decompress_safe function needs to know where the compressed data is, how many bytes long it is, + // where the regen_buffer memory location is, and how large regen_buffer (uncompressed) output will be. + // Again, save the return_value. + const int decompressed_size = LZ4_decompress_safe(compressed_data, regen_buffer, compressed_data_size, src_size); + free(compressed_data); /* no longer useful */ + if (decompressed_size < 0) + run_screaming("A negative result from LZ4_decompress_safe indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", decompressed_size); + if (decompressed_size == 0) run_screaming("I'm not sure this function can ever return 0. Documentation in lz4.h doesn't indicate so.", 1); - if (return_value > 0) + if (decompressed_size > 0) printf("We successfully decompressed some data!\n"); - // Not only does a positive return value mean success, the value returned == the number of bytes read from the compressed_data - // stream. I'm not sure there's ever a time you'll need to know this in most cases... + // Not only does a positive return value mean success, + // value returned == number of bytes regenerated from compressed_data stream. /* Validation */ // We should be able to compare our original *src with our *new_src and be byte-for-byte identical. - if (memcmp(src, new_src, src_size) != 0) + if (memcmp(src, regen_buffer, src_size) != 0) run_screaming("Validation failed. *src and *new_src are not identical.", 1); - printf("Validation done. The string we ended up with is:\n%s\n", new_src); + printf("Validation done. The string we ended up with is:\n%s\n", regen_buffer); return 0; } diff --git a/lib/Makefile b/lib/Makefile index 28242cc..dd33f50 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -3,6 +3,8 @@ # Copyright (C) Yann Collet 2011-2016 # All rights reserved. # +# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets +# # BSD license # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: @@ -38,22 +40,19 @@ LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCR LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) -LIBVER := $(shell echo $(LIBVER_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) -BUILD_STATIC:= yes +BUILD_STATIC:=yes -DESTDIR?= -PREFIX ?= /usr/local -CPPFLAGS= -DXXH_NAMESPACE=LZ4_ -CFLAGS ?= -O3 -CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ - -Wpointer-arith -Wstrict-aliasing=1 -CFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CPPFLAGS+= -DXXH_NAMESPACE=LZ4_ +CFLAGS ?= -O3 +DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ + -Wundef -Wpointer-arith -Wstrict-aliasing=1 +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -LIBDIR?= $(PREFIX)/lib -INCLUDEDIR=$(PREFIX)/include +SRCFILES := $(sort $(wildcard *.c)) # OS X linker doesn't support -soname, and use different extension @@ -62,7 +61,7 @@ ifeq ($(shell uname), Darwin) SHARED_EXT = dylib SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) - SONAME_FLAGS = -install_name $(PREFIX)/lib/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) + SONAME_FLAGS = -install_name $(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) else SONAME_FLAGS = -Wl,-soname=liblz4.$(SHARED_EXT).$(LIBVER_MAJOR) SHARED_EXT = so @@ -70,73 +69,109 @@ else SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif +LIBLZ4 = liblz4.$(SHARED_EXT_VER) + +.PHONY: default default: lib-release +lib-release: DEBUGFLAGS := +lib-release: lib + +lib: liblz4.a liblz4 + all: lib all32: CFLAGS+=-m32 all32: all -lib: liblz4.a liblz4 - -lib-release: CFLAGS := -O3 -lib-release: lib - -liblz4.a: *.c -ifeq ($(BUILD_STATIC),yes) +liblz4.a: $(SRCFILES) +ifeq ($(BUILD_STATIC),yes) # can be disabled on command line @echo compiling static library - @$(CC) $(FLAGS) -c $^ + @$(CC) $(CPPFLAGS) $(CFLAGS) -c $^ @$(AR) rcs $@ *.o endif -liblz4: *.c +$(LIBLZ4): $(SRCFILES) @echo compiling dynamic library $(LIBVER) ifneq (,$(filter Windows%,$(OS))) @$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib else - @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) + @$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@ @echo creating versioned links - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR) - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT) + @ln -sf $@ liblz4.$(SHARED_EXT_MAJOR) + @ln -sf $@ liblz4.$(SHARED_EXT) endif +liblz4: $(LIBLZ4) + clean: - @$(RM) core *.o *.a *.$(SHARED_EXT) liblz4.pc dll/liblz4.dll dll/liblz4.lib + @$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib + @$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER) @echo Cleaning library completed -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#----------------------------------------------------------------------------- +# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) + +DESTDIR ?= +# directory variables : GNU conventions prefer lowercase +# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html +# support both lower and uppercase (BSD), use uppercase in script +prefix ?= /usr/local +PREFIX ?= $(prefix) +exec_prefix ?= $(PREFIX) +libdir ?= $(exec_prefix)/lib +LIBDIR ?= $(libdir) +includedir ?= $(PREFIX)/include +INCLUDEDIR ?= $(includedir) + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly)) +PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig +else +PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig +endif + +ifneq (,$(filter $(shell uname),SunOS)) +INSTALL ?= ginstall +else +INSTALL ?= install +endif + +INSTALL_PROGRAM ?= $(INSTALL) +INSTALL_DATA ?= $(INSTALL) -m 644 liblz4.pc: liblz4.pc.in Makefile @echo creating pkgconfig @sed -e 's|@PREFIX@|$(PREFIX)|' \ - -e 's|@LIBDIR@|$(LIBDIR)|' \ - -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ - -e 's|@VERSION@|$(LIBVER)|' \ - $< >$@ + -e 's|@LIBDIR@|$(LIBDIR)|' \ + -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ + -e 's|@VERSION@|$(LIBVER)|' \ + $< >$@ install: lib liblz4.pc - @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ - @install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) - @install -m 644 liblz4.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ + @$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ $(DESTDIR)$(LIBDIR)/ + @$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/ + @echo Installing libraries ifeq ($(BUILD_STATIC),yes) - @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a + @$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a + @$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h endif - @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h - @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h - @echo lz4 static and shared libraries installed + @$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) + @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) + @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) + @echo Installing headers in $(INCLUDEDIR) + @$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h + @$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h + @echo lz4 libraries installed uninstall: + @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a @$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h diff --git a/lib/README.md b/lib/README.md index 18793d4..fc5d4e9 100644 --- a/lib/README.md +++ b/lib/README.md @@ -1,44 +1,43 @@ LZ4 - Library Files ================================ -The directory contains many files, but depending on project's objectives, +The `/lib` directory contains many files, but depending on project's objectives, not all of them are necessary. #### Minimal LZ4 build The minimum required is **`lz4.c`** and **`lz4.h`**, -which will provide the fast compression and decompression algorithm. +which provides the fast compression and decompression algorithm. +They generate and decode data using [LZ4 block format]. -#### The High Compression variant of LZ4 +#### High Compression variant -For more compression at the cost of compression speed, -the High Compression variant **lz4hc** is available. -It's necessary to add **`lz4hc.c`** and **`lz4hc.h`**. -The variant still depends on regular `lz4` source files. -In particular, the decompression is still provided by `lz4.c`. +For more compression ratio at the cost of compression speed, +the High Compression variant called **lz4hc** is available. +Add files **`lz4hc.c`**, **`lz4hc.h`** and **`lz4opt.h`**. +The variant still depends on regular `lib/lz4.*` source files. -#### Compatibility issues +#### Frame variant, for interoperability -In order to produce files or streams compatible with `lz4` command line utility, +In order to produce compressed data compatible with `lz4` command line utility, it's necessary to encode lz4-compressed blocks using the [official interoperable frame format]. This format is generated and decoded automatically by the **lz4frame** library. -In order to work properly, lz4frame needs lz4 and lz4hc, and also **xxhash**, -which provides error detection. -(_Advanced stuff_ : It's possible to hide xxhash symbols into a local namespace. -This is what `liblz4` does, to avoid symbol duplication -in case a user program would link to several libraries containing xxhash symbols.) +Its public API is described in `lib/lz4frame.h`. +In order to work properly, lz4frame needs all other modules present in `/lib`, +including, lz4 and lz4hc, and also **xxhash**. +So it's necessary to include all `*.c` and `*.h` files present in `/lib`. -#### Advanced API +#### Advanced / Experimental API -A more complex `lz4frame_static.h` is also provided. -It contains definitions which are not guaranteed to remain stable within future versions. -It must be used with static linking ***only***. +A complex API defined in `lz4frame_static.h` contains definitions +which are not guaranteed to remain stable in future versions. +As a consequence, it must be used with static linking ***only***. -#### Using MinGW+MSYS to create DLL +#### Windows : using MinGW+MSYS to create DLL DLL can be created using MinGW+MSYS with the `make liblz4` command. This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`. @@ -47,27 +46,28 @@ The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library `dll\liblz4.dll` are required to compile a project using gcc/MinGW. The dynamic library has to be added to linking options. It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be compiled with "liblz4.lib". For example: +file it should be linked with `dll\liblz4.dll`. For example: ``` gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll ``` -The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. +The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. -#### Miscellaneous +#### Miscellaneous Other files present in the directory are not source code. There are : - - LICENSE : contains the BSD license text - - Makefile : script to compile or install lz4 library (static or dynamic) - - liblz4.pc.in : for pkg-config (make install) - - README.md : this file + - `LICENSE` : contains the BSD license text + - `Makefile` : `make` script to compile and install lz4 library (static and dynamic) + - `liblz4.pc.in` : for `pkg-config` (used in `make install`) + - `README.md` : this file [official interoperable frame format]: ../doc/lz4_Frame_format.md +[LZ4 block format]: ../doc/lz4_Block_format.md -#### License +#### License All source material within __lib__ directory are BSD 2-Clause licensed. See [LICENSE](LICENSE) for details. -The license is also repeated at the top of each source file. +The license is also reminded at the top of each source file. diff --git a/lib/dll/example/README.md b/lib/dll/example/README.md index 4908663..223e473 100644 --- a/lib/dll/example/README.md +++ b/lib/dll/example/README.md @@ -39,7 +39,7 @@ The header files from `include\` and the dynamic library `dll\liblz4.dll` are required to compile a project using gcc/MinGW. The dynamic library has to be added to linking options. It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be compiled with "liblz4.dll". For example: +file it should be linked with `dll\liblz4.dll`. For example: ``` gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\liblz4.dll ``` @@ -1,6 +1,6 @@ /* LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2016, Yann Collet. + Copyright (C) 2011-2017, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -37,12 +37,12 @@ * Tuning parameters **************************************/ /* - * HEAPMODE : + * LZ4_HEAPMODE : * Select how default compression functions will allocate memory for their hash table, * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). */ -#ifndef HEAPMODE -# define HEAPMODE 0 +#ifndef LZ4_HEAPMODE +# define LZ4_HEAPMODE 0 #endif /* @@ -63,16 +63,15 @@ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets which generate assembly depending on alignment. + * It can generate buggy code on targets which assembly generation depends on alignment. * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. * Prefer these methods in priority order (0 > 1 > 2) */ -#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define LZ4_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# elif defined(__INTEL_COMPILER) || defined(__GNUC__) # define LZ4_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -86,6 +85,7 @@ #endif + /*-************************************ * Dependency **************************************/ @@ -97,20 +97,49 @@ * Compiler Options **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # include <intrin.h> # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ -#else -# if defined(__GNUC__) || defined(__clang__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define FORCE_INLINE static inline -# else -# define FORCE_INLINE static -# endif #endif /* _MSC_VER */ +#ifndef LZ4_FORCE_INLINE +# ifdef _MSC_VER /* Visual Studio */ +# define LZ4_FORCE_INLINE static __forceinline +# else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define LZ4_FORCE_INLINE static inline +# endif +# else +# define LZ4_FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +# endif /* _MSC_VER */ +#endif /* LZ4_FORCE_INLINE */ + +/* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE + * Gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy, + * together with a simple 8-byte copy loop as a fall-back path. + * However, this optimization hurts the decompression speed by >30%, + * because the execution does not go to the optimized loop + * for typical compressible data, and all of the preamble checks + * before going to the fall-back path become useless overhead. + * This optimization happens only with the -O3 flag, and -O2 generates + * a simple 8-byte copy loop. + * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy + * functions are annotated with __attribute__((optimize("O2"))), + * and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute + * of LZ4_wildCopy does not affect the compression speed. + */ +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) +# define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2"))) +# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE +#else +# define LZ4_FORCE_O2_GCC_PPC64LE +# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static +#endif + #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else @@ -247,7 +276,8 @@ static void LZ4_copy8(void* dst, const void* src) } /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ -static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +LZ4_FORCE_O2_INLINE_GCC_PPC64LE +void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; const BYTE* s = (const BYTE*)srcPtr; @@ -281,15 +311,36 @@ static const int LZ4_minLength = (MFLIMIT+1); /*-************************************ -* Common Utils +* Error detection **************************************/ -#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1) +# include <assert.h> +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include <stdio.h> +static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif /*-************************************ * Common functions **************************************/ -static unsigned LZ4_NbCommonBytes (register reg_t val) +static unsigned LZ4_NbCommonBytes (reg_t val) { if (LZ4_isLittleEndian()) { if (sizeof(val)==8) { @@ -300,7 +351,14 @@ static unsigned LZ4_NbCommonBytes (register reg_t val) # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctzll((U64)val) >> 3); # else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, + 0, 3, 1, 3, 1, 4, 2, 7, + 0, 2, 3, 6, 1, 5, 3, 5, + 1, 3, 4, 4, 2, 5, 6, 7, + 7, 0, 1, 2, 3, 3, 4, 6, + 2, 6, 5, 5, 3, 4, 5, 6, + 7, 1, 2, 4, 6, 4, 4, 5, + 7, 2, 6, 5, 7, 6, 7, 7 }; return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; # endif } else /* 32 bits */ { @@ -311,12 +369,15 @@ static unsigned LZ4_NbCommonBytes (register reg_t val) # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctz((U32)val) >> 3); # else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, + 3, 2, 2, 1, 3, 2, 0, 1, + 3, 3, 1, 2, 2, 2, 2, 0, + 3, 1, 2, 0, 1, 0, 1, 1 }; return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; # endif } } else /* Big Endian CPU */ { - if (sizeof(val)==8) { + if (sizeof(val)==8) { /* 64-bits */ # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r = 0; _BitScanReverse64( &r, val ); @@ -324,8 +385,11 @@ static unsigned LZ4_NbCommonBytes (register reg_t val) # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_clzll((U64)val) >> 3); # else + static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. + Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. + Note that this code path is never triggered in 32-bits mode. */ unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } r += (!val); return r; @@ -348,11 +412,20 @@ static unsigned LZ4_NbCommonBytes (register reg_t val) } #define STEPSIZE sizeof(reg_t) -static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +LZ4_FORCE_INLINE +unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) { const BYTE* const pStart = pIn; - while (likely(pIn<pInLimit-(STEPSIZE-1))) { + if (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { + pIn+=STEPSIZE; pMatch+=STEPSIZE; + } else { + return LZ4_NbCommonBytes(diff); + } } + + while (likely(pIn < pInLimit-(STEPSIZE-1))) { reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } pIn += LZ4_NbCommonBytes(diff); @@ -391,6 +464,7 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive; * Local Utils **************************************/ int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } int LZ4_sizeofState() { return LZ4_STREAMSIZE; } @@ -417,7 +491,7 @@ static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); } -FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) +LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) { if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); return LZ4_hash4(LZ4_read32(p), tableType); @@ -433,7 +507,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy } } -FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); @@ -446,7 +520,7 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tab { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); @@ -455,7 +529,7 @@ FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableTy /** LZ4_compress_generic() : inlined, to ensure branches are decided at compilation time */ -FORCE_INLINE int LZ4_compress_generic( +LZ4_FORCE_INLINE int LZ4_compress_generic( LZ4_stream_t_internal* const cctx, const char* const source, char* const dest, @@ -597,7 +671,11 @@ _next_match: *token += ML_MASK; matchCode -= ML_MASK; LZ4_write32(op, 0xFFFFFFFF); - while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; + while (matchCode >= 4*255) { + op+=4; + LZ4_write32(op, 0xFFFFFFFF); + matchCode -= 4*255; + } op += matchCode / 255; *op++ = (BYTE)(matchCode % 255); } else @@ -677,7 +755,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { -#if (HEAPMODE) +#if (LZ4_HEAPMODE) void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else LZ4_stream_t ctx; @@ -686,7 +764,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); -#if (HEAPMODE) +#if (LZ4_HEAPMODE) FREEMEM(ctxPtr); #endif return result; @@ -890,7 +968,7 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { -#if (HEAPMODE) +#if (LZ4_HEAPMODE) LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else LZ4_stream_t ctxBody; @@ -899,7 +977,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); -#if (HEAPMODE) +#if (LZ4_HEAPMODE) FREEMEM(ctx); #endif return result; @@ -921,11 +999,13 @@ LZ4_stream_t* LZ4_createStream(void) void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { + DEBUGLOG(4, "LZ4_resetStream"); MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { + if (!LZ4_stream) return 0; /* support free on NULL */ FREEMEM(LZ4_stream); return (0); } @@ -1080,47 +1160,46 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) * Decompression functions *******************************/ /*! LZ4_decompress_generic() : - * This generic decompression function cover all use cases. - * It shall be instantiated several times, using different sets of directives - * Note that it is important this generic function is really inlined, + * This generic decompression function covers all use cases. + * It shall be instantiated several times, using different sets of directives. + * Note that it is important for performance that this function really get inlined, * in order to remove useless branches during compilation optimization. */ -FORCE_INLINE int LZ4_decompress_generic( - const char* const source, - char* const dest, - int inputSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ +LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_INLINE int LZ4_decompress_generic( + const char* const src, + char* const dst, + int srcSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ int endOnInput, /* endOnOutputSize, endOnInputSize */ int partialDecoding, /* full, partial */ int targetOutputSize, /* only used if partialDecoding==partial */ int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest when no prefix */ + const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ const BYTE* const dictStart, /* only if dict==usingExtDict */ const size_t dictSize /* note : = 0 if noDict */ ) { - /* Local Variables */ - const BYTE* ip = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; - BYTE* op = (BYTE*) dest; + BYTE* op = (BYTE*) dst; BYTE* const oend = op + outputSize; BYTE* cpy; BYTE* oexit = op + targetOutputSize; - const BYTE* const lowLimit = lowPrefix - dictSize; const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; - const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; + const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; + const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; const int safeDecode = (endOnInput==endOnInputSize); const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); /* Special cases */ - if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ - if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => just decode everything */ + if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); /* Main Loop : decode sequences */ @@ -1129,8 +1208,27 @@ FORCE_INLINE int LZ4_decompress_generic( const BYTE* match; size_t offset; - /* get literal length */ unsigned const token = *ip++; + + /* shortcut for common case : + * in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes). + * this shortcut was tested on x86 and x64, where it improves decoding speed. + * it has not yet been benchmarked on ARM, Power, mips, etc. */ + if (((ip + 14 /*maxLL*/ + 2 /*offset*/ <= iend) + & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend)) + & ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) { + size_t const ll = token >> ML_BITS; + size_t const off = LZ4_readLE16(ip+ll); + const BYTE* const matchPtr = op + ll - off; /* pointer underflow risk ? */ + if ((off >= 18) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) { + size_t const ml = (token & ML_MASK) + MINMATCH; + memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/; + memcpy(op, matchPtr, 18); op += ml; + continue; + } + } + + /* decode literal length */ if ((length=(token>>ML_BITS)) == RUN_MASK) { unsigned s; do { @@ -1164,7 +1262,7 @@ FORCE_INLINE int LZ4_decompress_generic( /* get offset */ offset = LZ4_readLE16(ip); ip+=2; match = op - offset; - if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ /* get matchlength */ @@ -1208,14 +1306,13 @@ FORCE_INLINE int LZ4_decompress_generic( /* copy match within block */ cpy = op + length; if (unlikely(offset<8)) { - const int dec64 = dec64table[offset]; op[0] = match[0]; op[1] = match[1]; op[2] = match[2]; op[3] = match[3]; - match += dec32table[offset]; + match += inc32table[offset]; memcpy(op+4, match, 4); - match -= dec64; + match -= dec64table[offset]; } else { LZ4_copy8(op, match); match+=8; } op += 8; @@ -1232,31 +1329,34 @@ FORCE_INLINE int LZ4_decompress_generic( LZ4_copy8(op, match); if (length>16) LZ4_wildCopy(op+8, match+8, cpy); } - op=cpy; /* correction */ + op = cpy; /* correction */ } /* end of decoding */ if (endOnInput) - return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ else - return (int) (((const char*)ip)-source); /* Nb of input bytes read */ + return (int) (((const char*)ip)-src); /* Nb of input bytes read */ /* Overflow error detected */ _output_error: - return (int) (-(((const char*)ip)-source))-1; + return (int) (-(((const char*)ip)-src))-1; } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); @@ -1265,11 +1365,6 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize) /*===== streaming decompression functions =====*/ -/* - * If you prefer dynamic allocation methods, - * LZ4_createStreamDecode() - * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure. - */ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); @@ -1278,6 +1373,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void) int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { + if (!LZ4_stream) return 0; /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } @@ -1306,6 +1402,7 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti If it's not possible, save the relevant part of decoded data into a safe buffer, and indicate where it stands using LZ4_setStreamDecode() */ +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; @@ -1332,6 +1429,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch return result; } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; @@ -1366,7 +1464,8 @@ Advanced decoding functions : the dictionary must be explicitly provided within parameters */ -FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) { if (dictSize==0) return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); @@ -1378,17 +1477,20 @@ FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); } /* debug function */ +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); @@ -1,7 +1,7 @@ /* * LZ4 - Fast LZ compression algorithm * Header File - * Copyright (C) 2011-2016, Yann Collet. + * Copyright (C) 2011-2017, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -32,13 +32,13 @@ - LZ4 homepage : http://www.lz4.org - LZ4 source repository : https://github.com/lz4/lz4 */ -#ifndef LZ4_H_2983827168210 -#define LZ4_H_2983827168210 - #if defined (__cplusplus) extern "C" { #endif +#ifndef LZ4_H_2983827168210 +#define LZ4_H_2983827168210 + /* --- Dependency --- */ #include <stddef.h> /* size_t */ @@ -72,20 +72,28 @@ extern "C" { /* * LZ4_DLL_EXPORT : * Enable exporting of functions when building a Windows DLL +* LZ4LIB_VISIBILITY : +* Control library symbols visibility. */ +#ifndef LZ4LIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4LIB_VISIBILITY +# endif +#endif #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4LIB_API __declspec(dllexport) +# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define LZ4LIB_API +# define LZ4LIB_API LZ4LIB_VISIBILITY #endif - -/*========== Version =========== */ +/*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) @@ -94,8 +102,8 @@ extern "C" { #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) -LZ4LIB_API int LZ4_versionNumber (void); -LZ4LIB_API const char* LZ4_versionString (void); +LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; to be used when checking dll version */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; to be used when checking dll version */ /*-************************************ @@ -108,37 +116,38 @@ LZ4LIB_API const char* LZ4_versionString (void); * Reduced memory usage can improve speed, due to cache effect * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define LZ4_MEMORY_USAGE 14 - +#ifndef LZ4_MEMORY_USAGE +# define LZ4_MEMORY_USAGE 14 +#endif /*-************************************ * Simple Functions **************************************/ /*! LZ4_compress_default() : - Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). + Compresses 'srcSize' bytes from buffer 'src' + into already allocated 'dst' buffer of size 'dstCapacity'. + Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, + If the function cannot compress 'src' into a limited 'dst' budget, compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails */ -LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); + As a consequence, 'dst' content is not valid. + This function never writes outside 'dst' buffer, nor read outside 'source' buffer. + srcSize : supported max value is LZ4_MAX_INPUT_VALUE + dstCapacity : full or partial size of buffer 'dst' (which must be already allocated) + return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + or 0 if compression fails */ +LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_safe() : - compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). + compressedSize : is the exact complete size of the compressed block. + dstCapacity : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + If destination buffer is not large enough, decoding will stop and output an error code (negative value). If the source stream is detected malformed, the function will stop decoding and return a negative result. This function is protected against buffer overflow exploits, including malicious data packets. It never writes outside output buffer, nor reads outside input buffer. */ -LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); +LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); /*-************************************ @@ -167,7 +176,7 @@ LZ4_compress_fast() : An acceleration value of "1" is the same as regular LZ4_compress_default() Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. */ -LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); +LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! @@ -178,49 +187,49 @@ LZ4_compress_fast_extState() : Then, provide it as 'void* state' to compression function. */ LZ4LIB_API int LZ4_sizeofState(void); -LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_compress_destSize() : - Reverse the logic, by compressing as much data as possible from 'source' buffer - into already allocated buffer 'dest' of size 'targetDestSize'. - This function either compresses the entire 'source' content into 'dest' if it's large enough, - or fill 'dest' buffer completely with as much data as possible from 'source'. - *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. - New value is necessarily <= old value. - return : Nb bytes written into 'dest' (necessarily <= targetDestSize) - or 0 if compression fails + Reverse the logic : compresses as much data as possible from 'src' buffer + into already allocated buffer 'dst' of size 'targetDestSize'. + This function either compresses the entire 'src' content into 'dst' if it's large enough, + or fill 'dst' buffer completely with as much data as possible from 'src'. + *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + New value is necessarily <= old value. + return : Nb bytes written into 'dst' (necessarily <= targetDestSize) + or 0 if compression fails */ -LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); +LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); /*! -LZ4_decompress_fast() : - originalSize : is the original and therefore uncompressed size +LZ4_decompress_fast() : (unsafe!!) + originalSize : is the original uncompressed size return : the number of bytes read from the source buffer (in other words, the compressed size) If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. + Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes. + note : This function respects memory boundaries for *properly formed* compressed data. It is a bit faster than LZ4_decompress_safe(). However, it does not provide any protection against intentionally modified data stream (malicious input). Use this function in trusted environment only (data to decode comes from a trusted source). */ -LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); +LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); /*! LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. + This function decompress a compressed block of size 'srcSize' at position 'src' + into destination buffer 'dst' of size 'dstCapacity'. + The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that. + However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity). + @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity) + Note : this number can be < 'targetOutputSize' should the compressed block contain less data. Always control how many bytes were decoded. If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets. */ -LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); +LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); /*-********************************************* @@ -237,33 +246,38 @@ LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); /*! LZ4_resetStream() : * An LZ4_stream_t structure can be allocated once and re-used multiple times. - * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. + * Use this function to start compressing a new stream. */ LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); /*! LZ4_loadDict() : - * Use this function to load a static dictionary into LZ4_stream. + * Use this function to load a static dictionary into LZ4_stream_t. * Any previous data will be forgotten, only 'dictionary' will remain in memory. - * Loading a size of 0 is allowed. - * Return : dictionary size, in bytes (necessarily <= 64 KB) + * Loading a size of 0 is allowed, and is the same as reset. + * @return : dictionary size, in bytes (necessarily <= 64 KB) */ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); /*! LZ4_compress_fast_continue() : - * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. - * Important : Previous data blocks are assumed to still be present and unmodified ! + * Compress content into 'src' using data from previously compressed blocks, improving compression ratio. * 'dst' buffer must be already allocated. - * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. + * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * + * Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory ! + * Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB. + * Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * + * @return : size of compressed block + * or 0 if there is an error (typically, compressed data cannot fit into 'dst') + * After an error, the stream status is invalid, it can only be reset or freed. */ -LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); +LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its memory location, + * If previously compressed data block is not guaranteed to remain available at its current memory location, * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + * Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable. + * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. */ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); @@ -274,36 +288,41 @@ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dict ************************************************/ typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ -/* creation / destruction of streaming decompression tracking structure */ +/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : + * creation / destruction of streaming decompression tracking structure. + * A tracking structure can be re-used multiple times sequentially. */ LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); /*! LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * Setting a size of 0 is allowed (same effect as reset). - * @return : 1 if OK, 0 if error + * An LZ4_streamDecode_t structure can be allocated once and re-used multiple times. + * Use this function to start decompression of a new stream of blocks. + * A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order. + * @return : 1 if OK, 0 if error */ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); -/*! -LZ4_decompress_*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - In the case of a ring buffers, decoding buffer must be either : - - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including small ones ( < 64 KB). - - _At least_ 64 KB + 8 bytes + maxBlockSize. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including larger than decoding buffer. - Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - and indicate where it is saved using LZ4_setStreamDecode() +/*! LZ4_decompress_*_continue() : + * These decoding functions allow decompression of consecutive blocks in "streaming" mode. + * A block is an unsplittable entity, it must be presented entirely to a decompression function. + * Decompression functions only accept one block at a time. + * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB). + * + * Special : if application sets a ring buffer for decompression, it must respect one of the following conditions : + * - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + * In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + * maxBlockSize is implementation dependent. It's the maximum size of any single block. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including small ones ( < 64 KB). + * - _At least_ 64 KB + 8 bytes + maxBlockSize. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including larger than decoding buffer. + * Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + * and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block. */ -LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); +LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); +LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); /*! LZ4_decompress_*_usingDict() : @@ -311,8 +330,8 @@ LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecod * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() * They are stand-alone, and don't need an LZ4_streamDecode_t structure. */ -LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); +LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); +LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); /*^********************************************** @@ -374,7 +393,7 @@ typedef struct { * init this structure before first use. * note : only use in association with static linking ! * this definition is not API/ABI safe, - * and may change in a future version ! + * it may change in a future version ! */ #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) @@ -400,11 +419,12 @@ union LZ4_streamDecode_u { } ; /* previously typedef'd to LZ4_streamDecode_t */ -/*=************************************ +/*-************************************ * Obsolete Functions **************************************/ -/* Deprecation warnings */ -/* Should these warnings be a problem, + +/*! Deprecation warnings + Should deprecation warnings be a problem, it is generally possible to disable them, typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. @@ -413,9 +433,11 @@ union LZ4_streamDecode_u { # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ #else # define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */ +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) +# elif (LZ4_GCC_VERSION >= 405) # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) # elif (LZ4_GCC_VERSION >= 301) # define LZ4_DEPRECATED(message) __attribute__((deprecated)) @@ -428,36 +450,30 @@ union LZ4_streamDecode_u { #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ /* Obsolete compression functions */ -LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* Obsolete decompression functions */ -/* These function names are completely deprecated and must no longer be used. - They are only provided in lz4.c for compatibility with older programs. - - LZ4_uncompress is the same as LZ4_decompress_fast - - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe - These function prototypes are now disabled; uncomment them only if you really need them. - It is highly recommended to stop using these prototypes and migrate to maintained ones */ -/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ -/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ +LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); /* Obsolete streaming functions; use new streaming interface whenever possible */ -LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); -LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); -LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); -LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); /* Obsolete streaming decoding functions */ -LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); -LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + +#endif /* LZ4_H_2983827168210 */ #if defined (__cplusplus) } #endif - -#endif /* LZ4_H_2983827168210 */ diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 3c2b788..488ab75 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -61,15 +61,16 @@ You can contact the author at : **************************************/ #include "lz4frame_static.h" #include "lz4.h" +#define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" /*-************************************ -* Common Utils +* Debug **************************************/ -#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +#define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ /*-************************************ @@ -158,7 +159,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB static const size_t minFHSize = 7; -static const size_t maxFHSize = 15; /* max Frame Header Size */ +static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ static const size_t BHSize = 4; @@ -170,6 +171,7 @@ typedef struct LZ4F_cctx_s LZ4F_preferences_t prefs; U32 version; U32 cStage; + const LZ4F_CDict* cdict; size_t maxBlockSize; size_t maxBufferSize; BYTE* tmpBuff; @@ -178,7 +180,7 @@ typedef struct LZ4F_cctx_s U64 totalInSize; XXH32_state_t xxh; void* lz4CtxPtr; - U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ + U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ } LZ4F_cctx_t; @@ -209,12 +211,15 @@ LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult) static LZ4F_errorCode_t err0r(LZ4F_errorCodes code) { - LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); /* A compilation error here means sizeof(ptrdiff_t) is not large enough */ + /* A compilation error here means sizeof(ptrdiff_t) is not large enough */ + LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); return (LZ4F_errorCode_t)-(ptrdiff_t)code; } unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } +int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; } + /*-************************************ * Private functions @@ -241,7 +246,8 @@ static BYTE LZ4F_headerChecksum (const void* header, size_t length) /*-************************************ * Simple-pass compression functions **************************************/ -static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize) +static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, + const size_t srcSize) { LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB; size_t maxBlockSize = 64 KB; @@ -254,33 +260,44 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI return requestedBSID; } -static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered) +/*! LZ4F_compressBound_internal() : + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario. + * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + */ +static size_t LZ4F_compressBound_internal(size_t srcSize, + const LZ4F_preferences_t* preferencesPtr, + size_t alreadyBuffered) { LZ4F_preferences_t prefsNull; memset(&prefsNull, 0, sizeof(prefsNull)); prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; - size_t const blockSize = LZ4F_getBlockSize(bid); + U32 const flush = prefsPtr->autoFlush | (srcSize==0); + LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID; + size_t const blockSize = LZ4F_getBlockSize(blockID); size_t const maxBuffered = blockSize - 1; size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); size_t const maxSrcSize = srcSize + bufferedSize; unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); - size_t const partialBlockSize = srcSize & (blockSize-1); - size_t const lastBlockSize = prefsPtr->autoFlush ? partialBlockSize : 0; + size_t const partialBlockSize = (srcSize - (srcSize==0)) & (blockSize-1); /* 0 => -1 == MAX => blockSize-1 */ + size_t const lastBlockSize = flush ? partialBlockSize : 0; unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); - size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */ + size_t const blockHeaderSize = 4; + size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag; size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); - return (blockHeaderSize * nbBlocks) + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;; + return ((blockHeaderSize + blockCRCSize) * nbBlocks) + + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd; } } size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefs; - size_t const headerSize = maxFHSize; /* max header size, including magic number and frame content size */ + size_t const headerSize = maxFHSize; /* max header size, including optional fields */ if (preferencesPtr!=NULL) prefs = *preferencesPtr; else memset(&prefs, 0, sizeof(prefs)); @@ -290,30 +307,31 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere } -/*! LZ4F_compressFrame() : -* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step. -* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. -* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() -* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) -* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default. -* The result of the function is the number of bytes written into dstBuffer. -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -*/ -size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +/*! LZ4F_compressFrame_usingCDict() : + * Compress srcBuffer using a dictionary, in a single step. + * cdict can be NULL, in which case, no dictionary is used. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, + * however, it's the only way to provide a dictID, so it's not recommended. + * @return : number of bytes written into dstBuffer, + * or an error code if it fails (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) { LZ4F_cctx_t cctxI; - LZ4_stream_t lz4ctx; + LZ4_stream_t lz4ctx; /* pretty large on stack */ LZ4F_preferences_t prefs; LZ4F_compressOptions_t options; BYTE* const dstStart = (BYTE*) dstBuffer; BYTE* dstPtr = dstStart; BYTE* const dstEnd = dstStart + dstCapacity; - memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */ - memset(&options, 0, sizeof(options)); - + memset(&cctxI, 0, sizeof(cctxI)); cctxI.version = LZ4F_VERSION; - cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ + cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */ if (preferencesPtr!=NULL) prefs = *preferencesPtr; @@ -322,22 +340,23 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (prefs.frameInfo.contentSize != 0) prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - if (prefs.compressionLevel < LZ4HC_MIN_CLEVEL) { - cctxI.lz4CtxPtr = &lz4ctx; - cctxI.lz4CtxLevel = 1; - } - prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) - prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* no need for linked blocks */ + prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */ + if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { + cctxI.lz4CtxPtr = &lz4ctx; + cctxI.lz4CtxLevel = 1; + } /* fast compression context pre-created on stack */ + + memset(&options, 0, sizeof(options)); options.stableSrc = 1; - if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) + if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */ return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - { size_t const headerSize = LZ4F_compressBegin(&cctxI, dstBuffer, dstCapacity, &prefs); /* write header */ + { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs); /* write header */ if (LZ4F_isError(headerSize)) return headerSize; dstPtr += headerSize; /* header size */ } @@ -349,24 +368,91 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (LZ4F_isError(tailSize)) return tailSize; dstPtr += tailSize; } - if (prefs.compressionLevel >= LZ4HC_MIN_CLEVEL) /* no allocation done with lz4 fast */ + if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* Ctx allocation only for lz4hc */ FREEMEM(cctxI.lz4CtxPtr); return (dstPtr - dstStart); } +/*! LZ4F_compressFrame() : + * Compress an entire srcBuffer into a valid LZ4 frame, in a single step. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity, + srcBuffer, srcSize, + NULL, preferencesPtr); +} + + +/*-*************************************************** +* Dictionary compression +*****************************************************/ + +struct LZ4F_CDict_s { + void* dictContent; + LZ4_stream_t* fastCtx; + LZ4_streamHC_t* HCCtx; +}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */ + +/*! LZ4F_createCDict() : + * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict + * @return : digested dictionary for compression, or NULL if failed */ +LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) +{ + const char* dictStart = (const char*)dictBuffer; + LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict)); + if (!cdict) return NULL; + if (dictSize > 64 KB) { + dictStart += dictSize - 64 KB; + dictSize = 64 KB; + } + cdict->dictContent = ALLOCATOR(dictSize); + cdict->fastCtx = LZ4_createStream(); + cdict->HCCtx = LZ4_createStreamHC(); + if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) { + LZ4F_freeCDict(cdict); + return NULL; + } + memcpy(cdict->dictContent, dictStart, dictSize); + LZ4_resetStream(cdict->fastCtx); + LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize); + LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT); + LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize); + return cdict; +} + +void LZ4F_freeCDict(LZ4F_CDict* cdict) +{ + if (cdict==NULL) return; /* support free on NULL */ + FREEMEM(cdict->dictContent); + LZ4_freeStream(cdict->fastCtx); + LZ4_freeStreamHC(cdict->HCCtx); + FREEMEM(cdict); +} + + /*-********************************* * Advanced compression functions ***********************************/ /*! LZ4F_createCompressionContext() : - * The first thing to do is to create a compressionContext object, which will be used in all compression operations. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. - * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. - * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. - * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. - * Object can release its memory using LZ4F_freeCompressionContext(); + * The first thing to do is to create a compressionContext object, which will be used in all compression operations. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. + * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries. + * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. + * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. + * Object can release its memory using LZ4F_freeCompressionContext(); */ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) { @@ -374,7 +460,7 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_c if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed); cctxPtr->version = version; - cctxPtr->cStage = 0; /* Next stage : write header */ + cctxPtr->cStage = 0; /* Next stage : init stream */ *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; @@ -386,8 +472,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp { LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext; - if (cctxPtr != NULL) { /* null pointers can be safely provided to this function, like free() */ - FREEMEM(cctxPtr->lz4CtxPtr); + if (cctxPtr != NULL) { /* support free on NULL */ + FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */ FREEMEM(cctxPtr->tmpBuff); FREEMEM(LZ4F_compressionContext); } @@ -396,59 +482,77 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp } -/*! LZ4F_compressBegin() : - * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. +/*! LZ4F_compressBegin_usingCDict() : + * init streaming compression and writes frame header into dstBuffer. + * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. * @return : number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr) +size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefNull; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; BYTE* headerStart; - size_t requiredBuffSize; if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - if (cctxPtr->cStage != 0) return err0r(LZ4F_ERROR_GENERIC); memset(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; cctxPtr->prefs = *preferencesPtr; - /* ctx Management */ - { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ - if (cctxPtr->lz4CtxLevel < tableID) { + /* Ctx Management */ + { U32 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ + if (cctxPtr->lz4CtxLevel < ctxTypeID) { FREEMEM(cctxPtr->lz4CtxPtr); - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); else cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); - cctxPtr->lz4CtxLevel = tableID; - } - } + if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed); + cctxPtr->lz4CtxLevel = ctxTypeID; + } } /* Buffer Management */ - if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + if (cctxPtr->prefs.frameInfo.blockSizeID == 0) + cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); - requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB); - if (preferencesPtr->autoFlush) - requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB; /* just needs dict */ + { size_t const requiredBuffSize = preferencesPtr->autoFlush ? + (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB : /* only needs windows size */ + cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB); - if (cctxPtr->maxBufferSize < requiredBuffSize) { - cctxPtr->maxBufferSize = requiredBuffSize; - FREEMEM(cctxPtr->tmpBuff); - cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); - if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed); - } + if (cctxPtr->maxBufferSize < requiredBuffSize) { + cctxPtr->maxBufferSize = 0; + FREEMEM(cctxPtr->tmpBuff); + cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); + if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed); + cctxPtr->maxBufferSize = requiredBuffSize; + } } cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; XXH32_reset(&(cctxPtr->xxh), 0); - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) - LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); - else - LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); + + /* context init */ + cctxPtr->cdict = cdict; + if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) { + /* frame init only for blockLinked : blockIndependent will be init at each block */ + if (cdict) { + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { + memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx)); + } else { + memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx)); + LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + } + } else { + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) + LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); + else + LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); + } + } /* Magic Number */ LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); @@ -457,9 +561,11 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit /* FLG Byte */ *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ - + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ - + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ - + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */ + + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) + + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4) + + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3) + + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) + + (cctxPtr->prefs.frameInfo.dictID > 0) ); /* BD Byte */ *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); /* Optional Frame content size field */ @@ -468,16 +574,36 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit dstPtr += 8; cctxPtr->totalInSize = 0; } - /* CRC Byte */ + /* Optional dictionary ID field */ + if (cctxPtr->prefs.frameInfo.dictID) { + LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID); + dstPtr += 4; + } + /* Header CRC Byte */ *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart); dstPtr++; cctxPtr->cStage = 1; /* header written, now request input data block */ - return (dstPtr - dstStart); } +/*! LZ4F_compressBegin() : + * init streaming compression and writes frame header into dstBuffer. + * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. + * preferencesPtr can be NULL, in which case default parameters are selected. + * @return : number of bytes written into dstBuffer for the header + * or an error code (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressBegin_usingCDict(cctxPtr, dstBuffer, dstCapacity, + NULL, preferencesPtr); +} + + /* LZ4F_compressBound() : * @ return size of Dst buffer given a srcSize to handle worst case situations. * The LZ4F_frameInfo_t structure is optional : if NULL, preferences will be set to cover worst case situations. @@ -489,54 +615,80 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP } -typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level); +typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict); + -static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level) +/*! LZ4F_makeBlock(): + * compress a single block, add header and checksum + * assumption : dst buffer capacity is >= srcSize */ +static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, + compressFunc_t compress, void* lz4ctx, int level, + const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag) { - /* compress a single block */ BYTE* const cSizePtr = (BYTE*)dst; - U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); + U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), + (int)(srcSize), (int)(srcSize-1), + level, cdict); LZ4F_writeLE32(cSizePtr, cSize); if (cSize == 0) { /* compression failed */ cSize = (U32)srcSize; LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); memcpy(cSizePtr+4, src, srcSize); } - return cSize + 4; + if (crcFlag) { + U32 const crc32 = XXH32(cSizePtr+4, cSize, 0); /* checksum of compressed data */ + LZ4F_writeLE32(cSizePtr+4+cSize, crc32); + } + return 4 + cSize + ((U32)crcFlag)*4; } -static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level) +static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + int const acceleration = (level < -1) ? -level : 1; + if (cdict) { + memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx)); + return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); + } + return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration); +} + +static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { - (void) level; - return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, 1); + int const acceleration = (level < -1) ? -level : 1; + (void)cdict; /* init once at beginning of frame */ + return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); } -static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level) +static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { - (void) level; - return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, 1); + if (cdict) { + memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx)); + LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level); + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); + } + return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level); } -static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { - (void) level; - return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize); + (void)level; (void)cdict; /* init once at beginning of frame */ + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); } static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) { - if (level < LZ4HC_MIN_CLEVEL) { - if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; - return LZ4F_localLZ4_compress_limitedOutput_continue; + if (level < LZ4HC_CLEVEL_MIN) { + if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock; + return LZ4F_compressBlock_continue; } - if (blockMode == LZ4F_blockIndependent) return LZ4_compress_HC_extStateHC; - return LZ4F_localLZ4_compressHC_limitedOutput_continue; + if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC; + return LZ4F_compressBlockHC_continue; } static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) { - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); } @@ -544,15 +696,16 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; /*! LZ4F_compressUpdate() : -* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. -* The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. -* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) -* You can get the minimum value of dstCapacity by using LZ4F_compressBound() -* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. -* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -*/ -size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) { LZ4F_compressOptions_t cOptionsNull; size_t const blockSize = cctxPtr->maxBlockSize; @@ -584,7 +737,9 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); srcPtr += sizeToCopy; - dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; @@ -592,16 +747,20 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci } while ((size_t)(srcEnd - srcPtr) >= blockSize) { - /* compress full block */ + /* compress full blocks */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); srcPtr += blockSize; } if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { /* compress remaining input < blockSize */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); srcPtr = srcEnd; } @@ -610,7 +769,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci if (compressOptionsPtr->stableSrc) { cctxPtr->tmpIn = cctxPtr->tmpBuff; } else { - int realDictSize = LZ4F_localSaveDict(cctxPtr); + int const realDictSize = LZ4F_localSaveDict(cctxPtr); if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC); cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; } @@ -620,7 +779,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */ && !(cctxPtr->prefs.autoFlush)) { - int realDictSize = LZ4F_localSaveDict(cctxPtr); + int const realDictSize = LZ4F_localSaveDict(cctxPtr); cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; } @@ -641,13 +800,13 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci /*! LZ4F_flush() : -* Should you need to create compressed data immediately, without waiting for a block to be filled, -* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. -* The result of the function is the number of bytes written into dstBuffer -* (it can be zero, this means there was no data left within compressionContext) -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. -*/ + * Should you need to create compressed data immediately, without waiting for a block to be filled, + * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. + * The result of the function is the number of bytes written into dstBuffer + * (it can be zero, this means there was no data left within compressionContext) + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + */ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr) { BYTE* const dstStart = (BYTE*)dstBuffer; @@ -663,7 +822,9 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); /* compress tmp buffer */ - dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; cctxPtr->tmpInSize = 0; @@ -678,14 +839,14 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const /*! LZ4F_compressEnd() : -* When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). -* It will flush whatever data remained within compressionContext (like LZ4_flush()) -* but also properly finalize the frame, with an endMark and a checksum. -* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. -* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. -*/ + * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). + * It will flush whatever data remained within compressionContext (like LZ4_flush()) + * but also properly finalize the frame, with an endMark and a checksum. + * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. + */ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) { BYTE* const dstStart = (BYTE*)dstBuffer; @@ -720,10 +881,23 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, * Frame Decompression *****************************************************/ +typedef enum { + dstage_getFrameHeader=0, dstage_storeFrameHeader, + dstage_init, + dstage_getBlockHeader, dstage_storeBlockHeader, + dstage_copyDirect, dstage_getBlockChecksum, + dstage_getCBlock, dstage_storeCBlock, + dstage_decodeCBlock, dstage_decodeCBlock_intoDst, + dstage_decodeCBlock_intoTmp, dstage_flushOut, + dstage_getSuffix, dstage_storeSuffix, + dstage_getSFrameSize, dstage_storeSFrameSize, + dstage_skipSkippable +} dStage_t; + struct LZ4F_dctx_s { LZ4F_frameInfo_t frameInfo; U32 version; - U32 dStage; + dStage_t dStage; U64 frameRemainingSize; size_t maxBlockSize; size_t maxBufferSize; @@ -731,40 +905,41 @@ struct LZ4F_dctx_s { size_t tmpInSize; size_t tmpInTarget; BYTE* tmpOutBuffer; - const BYTE* dict; + const BYTE* dict; size_t dictSize; BYTE* tmpOut; size_t tmpOutSize; size_t tmpOutStart; XXH32_state_t xxh; - BYTE header[16]; + XXH32_state_t blockChecksum; + BYTE header[LZ4F_HEADER_SIZE_MAX]; }; /* typedef'd to LZ4F_dctx in lz4frame.h */ /*! LZ4F_createDecompressionContext() : -* Create a decompressionContext object, which will track all decompression operations. -* Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object. -* Object can later be released using LZ4F_freeDecompressionContext(). -* @return : if != 0, there was an error during context creation. -*/ + * Create a decompressionContext object, which will track all decompression operations. + * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object. + * Object can later be released using LZ4F_freeDecompressionContext(). + * @return : if != 0, there was an error during context creation. + */ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) { - LZ4F_dctx* const dctxPtr = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx)); - if (dctxPtr==NULL) return err0r(LZ4F_ERROR_GENERIC); + LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx)); + if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC); - dctxPtr->version = versionNumber; - *LZ4F_decompressionContextPtr = dctxPtr; + dctx->version = versionNumber; + *LZ4F_decompressionContextPtr = dctx; return LZ4F_OK_NoError; } -LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctxPtr) +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx) { LZ4F_errorCode_t result = LZ4F_OK_NoError; - if (dctxPtr != NULL) { /* can accept NULL input, like free() */ - result = (LZ4F_errorCode_t)dctxPtr->dStage; - FREEMEM(dctxPtr->tmpIn); - FREEMEM(dctxPtr->tmpOutBuffer); - FREEMEM(dctxPtr); + if (dctx != NULL) { /* can accept NULL input, like free() */ + result = (LZ4F_errorCode_t)dctx->dStage; + FREEMEM(dctx->tmpIn); + FREEMEM(dctx->tmpOutBuffer); + FREEMEM(dctx); } return result; } @@ -772,22 +947,18 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctxPtr) /*==--- Streaming Decompression operations ---==*/ -typedef enum { dstage_getHeader=0, dstage_storeHeader, - dstage_getCBlockSize, dstage_storeCBlockSize, - dstage_copyDirect, - dstage_getCBlock, dstage_storeCBlock, - dstage_decodeCBlock, dstage_decodeCBlock_intoDst, - dstage_decodeCBlock_intoTmp, dstage_flushOut, - dstage_getSuffix, dstage_storeSuffix, - dstage_getSFrameSize, dstage_storeSFrameSize, - dstage_skipSkippable -} dStage_t; +void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) +{ + dctx->dStage = dstage_getFrameHeader; + dctx->dict = NULL; + dctx->dictSize = 0; +} /*! LZ4F_headerSize() : -* @return : size of frame header -* or an error code, which can be tested using LZ4F_isError() -*/ + * @return : size of frame header + * or an error code, which can be tested using LZ4F_isError() + */ static size_t LZ4F_headerSize(const void* src, size_t srcSize) { /* minimal srcSize to determine header size */ @@ -797,244 +968,245 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize) if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8; /* control magic number */ - if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); + if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) + return err0r(LZ4F_ERROR_frameType_unknown); /* Frame Header Size */ { BYTE const FLG = ((const BYTE*)src)[4]; U32 const contentSizeFlag = (FLG>>3) & _1BIT; - return contentSizeFlag ? maxFHSize : minFHSize; + U32 const dictIDFlag = FLG & _1BIT; + return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); } } /*! LZ4F_decodeHeader() : - input : `src` points at the **beginning of the frame** - output : set internal values of dctx, such as - dctxPtr->frameInfo and dctxPtr->dStage. - Also allocates internal buffers. - @return : nb Bytes read from srcVoidPtr (necessarily <= srcSize) - or an error code (testable with LZ4F_isError()) -*/ -static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize) + * input : `src` points at the **beginning of the frame** + * output : set internal values of dctx, such as + * dctx->frameInfo and dctx->dStage. + * Also allocates internal buffers. + * @return : nb Bytes read from src (necessarily <= srcSize) + * or an error code (testable with LZ4F_isError()) + */ +static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize) { - BYTE FLG, BD; - unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID; + unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID; size_t frameHeaderSize; const BYTE* srcPtr = (const BYTE*)src; /* need to decode header to get frameInfo */ if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */ - memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); + memset(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo)); /* special case : skippable frames */ if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) { - dctxPtr->frameInfo.frameType = LZ4F_skippableFrame; - if (src == (void*)(dctxPtr->header)) { - dctxPtr->tmpInSize = srcSize; - dctxPtr->tmpInTarget = 8; - dctxPtr->dStage = dstage_storeSFrameSize; + dctx->frameInfo.frameType = LZ4F_skippableFrame; + if (src == (void*)(dctx->header)) { + dctx->tmpInSize = srcSize; + dctx->tmpInTarget = 8; + dctx->dStage = dstage_storeSFrameSize; return srcSize; } else { - dctxPtr->dStage = dstage_getSFrameSize; + dctx->dStage = dstage_getSFrameSize; return 4; } } /* control magic number */ - if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); - dctxPtr->frameInfo.frameType = LZ4F_frame; + if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) + return err0r(LZ4F_ERROR_frameType_unknown); + dctx->frameInfo.frameType = LZ4F_frame; /* Flags */ - FLG = srcPtr[4]; - version = (FLG>>6) & _2BITS; - blockMode = (FLG>>5) & _1BIT; - blockChecksumFlag = (FLG>>4) & _1BIT; - contentSizeFlag = (FLG>>3) & _1BIT; - contentChecksumFlag = (FLG>>2) & _1BIT; + { U32 const FLG = srcPtr[4]; + U32 const version = (FLG>>6) & _2BITS; + blockChecksumFlag = (FLG>>4) & _1BIT; + blockMode = (FLG>>5) & _1BIT; + contentSizeFlag = (FLG>>3) & _1BIT; + contentChecksumFlag = (FLG>>2) & _1BIT; + dictIDFlag = FLG & _1BIT; + /* validate */ + if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */ + if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */ + } /* Frame Header Size */ - frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize; + frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); if (srcSize < frameHeaderSize) { /* not enough input to fully decode frame header */ - if (srcPtr != dctxPtr->header) - memcpy(dctxPtr->header, srcPtr, srcSize); - dctxPtr->tmpInSize = srcSize; - dctxPtr->tmpInTarget = frameHeaderSize; - dctxPtr->dStage = dstage_storeHeader; + if (srcPtr != dctx->header) + memcpy(dctx->header, srcPtr, srcSize); + dctx->tmpInSize = srcSize; + dctx->tmpInTarget = frameHeaderSize; + dctx->dStage = dstage_storeFrameHeader; return srcSize; } - BD = srcPtr[5]; - blockSizeID = (BD>>4) & _3BITS; - - /* validate */ - if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */ - if (blockChecksumFlag != 0) return err0r(LZ4F_ERROR_blockChecksum_unsupported); /* Not supported for the time being */ - if (((FLG>>0)&_2BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */ - if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */ - if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */ - if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */ + { U32 const BD = srcPtr[5]; + blockSizeID = (BD>>4) & _3BITS; + /* validate */ + if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */ + if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */ + } /* check header */ - { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); - if (HC != srcPtr[frameHeaderSize-1]) return err0r(LZ4F_ERROR_headerChecksum_invalid); } + { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); + if (HC != srcPtr[frameHeaderSize-1]) + return err0r(LZ4F_ERROR_headerChecksum_invalid); + } /* save */ - dctxPtr->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; - dctxPtr->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag; - dctxPtr->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; - dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); + dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; + dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag; + dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag; + dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; + dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID); if (contentSizeFlag) - dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); - - /* init */ - if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); - - /* internal buffers allocation */ - { size_t const bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB); - if (bufferNeeded > dctxPtr->maxBufferSize) { /* tmp buffers too small */ - FREEMEM(dctxPtr->tmpIn); - dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize); - if (dctxPtr->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed); - FREEMEM(dctxPtr->tmpOutBuffer); - dctxPtr->maxBufferSize = 0; - dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded); - if (dctxPtr->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed); - dctxPtr->maxBufferSize = bufferNeeded; - } } - dctxPtr->tmpInSize = 0; - dctxPtr->tmpInTarget = 0; - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = 0; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; - dctxPtr->tmpOutStart = 0; - dctxPtr->tmpOutSize = 0; + dctx->frameRemainingSize = + dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); + if (dictIDFlag) + dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5); - dctxPtr->dStage = dstage_getCBlockSize; + dctx->dStage = dstage_init; return frameHeaderSize; } /*! LZ4F_getFrameInfo() : -* Decodes frame header information, such as blockSize. Usage is optional. -* The objective is to extract header information before receiving decompressed data, typically for allocation purposes. -* LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t. -* The number of bytes consumed from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). -* Decompression must resume from where it stopped (srcBuffer + *srcSizePtr) -* @return : hint of the better `srcSize` to use for next call to LZ4F_decompress, -* or an error code which can be tested using LZ4F_isError(). -*/ -LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameInfoPtr, + * This function extracts frame parameters (max blockSize, frame checksum, etc.). + * Usage is optional. Objective is to provide relevant information for allocation purposes. + * This function works in 2 situations : + * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process. + * Amount of input data provided must be large enough to successfully decode the frame header. + * A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum. + * - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx. + * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + * Decompression must resume from (srcBuffer + *srcSizePtr). + * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, + * or an error code which can be tested using LZ4F_isError() + * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped. + * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. + */ +LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr) { - if (dctxPtr->dStage > dstage_storeHeader) { /* note : requires dstage_* header related to be at beginning of enum */ + if (dctx->dStage > dstage_storeFrameHeader) { /* assumption : dstage_* header enum at beginning of range */ /* frameInfo already decoded */ size_t o=0, i=0; *srcSizePtr = 0; - *frameInfoPtr = dctxPtr->frameInfo; - return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL); /* returns : recommended nb of bytes for LZ4F_decompress() */ + *frameInfoPtr = dctx->frameInfo; + /* returns : recommended nb of bytes for LZ4F_decompress() */ + return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL); } else { - size_t nextSrcSize, o=0; - size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr); - if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; } - if (*srcSizePtr < hSize) { *srcSizePtr=0; return err0r(LZ4F_ERROR_frameHeader_incomplete); } - - *srcSizePtr = hSize; - nextSrcSize = LZ4F_decompress(dctxPtr, NULL, &o, srcBuffer, srcSizePtr, NULL); - if (dctxPtr->dStage <= dstage_storeHeader) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* should not happen, already checked */ - *frameInfoPtr = dctxPtr->frameInfo; - return nextSrcSize; - } -} - + if (dctx->dStage == dstage_storeFrameHeader) { + /* frame decoding already started, in the middle of header => automatic fail */ + *srcSizePtr = 0; + return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted); + } else { + size_t decodeResult; + size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr); + if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; } + if (*srcSizePtr < hSize) { + *srcSizePtr=0; + return err0r(LZ4F_ERROR_frameHeader_incomplete); + } -/* trivial redirector, for common prototype */ -static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) -{ - (void)dictStart; (void)dictSize; - return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize); + decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize); + if (LZ4F_isError(decodeResult)) { + *srcSizePtr = 0; + } else { + *srcSizePtr = decodeResult; + decodeResult = BHSize; /* block header size */ + } + *frameInfoPtr = dctx->frameInfo; + return decodeResult; + } } } -static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) +/* LZ4F_updateDict() : + * only used for LZ4F_blockLinked mode */ +static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) { - if (dctxPtr->dictSize==0) - dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ + if (dctx->dictSize==0) + dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ - if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) { /* dictionary continuity */ - dctxPtr->dictSize += dstSize; + if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity */ + dctx->dictSize += dstSize; return; } if (dstPtr - dstPtr0 + dstSize >= 64 KB) { /* dstBuffer large enough to become dictionary */ - dctxPtr->dict = (const BYTE*)dstPtr0; - dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize; + dctx->dict = (const BYTE*)dstPtr0; + dctx->dictSize = dstPtr - dstPtr0 + dstSize; return; } - if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) { - /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */ - dctxPtr->dictSize += dstSize; + if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) { + /* assumption : dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart */ + dctx->dictSize += dstSize; return; } if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ - size_t const preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - size_t copySize = 64 KB - dctxPtr->tmpOutSize; - const BYTE* const oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; - if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; + size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer; + size_t copySize = 64 KB - dctx->tmpOutSize; + const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; + if (dctx->tmpOutSize > 64 KB) copySize = 0; if (copySize > preserveSize) copySize = preserveSize; - memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize; return; } - if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { /* copy dst into tmp to complete dict */ - if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) { /* tmp buffer not large enough */ + if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */ + if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */ size_t const preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); - dctxPtr->dictSize = preserveSize; + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); + dctx->dictSize = preserveSize; } - memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize); - dctxPtr->dictSize += dstSize; + memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize); + dctx->dictSize += dstSize; return; } /* join dict & dest into tmp */ { size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ - if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize; - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); - memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dstSize; + if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize; + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); + memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize); + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dstSize; } } /*! LZ4F_decompress() : -* Call this function repetitively to regenerate data compressed within srcBuffer. -* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. -* -* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). -* -* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). -* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. -* Remaining data will have to be presented again in a subsequent invocation. -* -* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. -* Basically, it's the size of the current (or remaining) compressed block + header of next block. -* Respecting the hint provides some boost to performance, since it allows less buffer shuffling. -* Note that this is just a hint, it's always possible to any srcSize value. -* When a frame is fully decoded, @return will be 0. -* If decompression failed, @return is an error code which can be tested using LZ4F_isError(). -*/ -size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, + * Call this function repetitively to regenerate compressed data in srcBuffer. + * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer + * into dstBuffer of capacity *dstSizePtr. + * + * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + * + * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * If number of bytes read is < number of bytes provided, then decompression operation is not complete. + * Remaining data will have to be presented again in a subsequent invocation. + * + * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling. + * Note that this is just a hint, and it's always possible to any srcSize value. + * When a frame is fully decoded, @return will be 0. + * If decompression failed, @return is an error code which can be tested using LZ4F_isError(). + */ +size_t LZ4F_decompress(LZ4F_dctx* dctx, void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, const LZ4F_decompressOptions_t* decompressOptionsPtr) @@ -1056,219 +1228,294 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, *srcSizePtr = 0; *dstSizePtr = 0; - /* programmed as a state machine */ + /* behaves as a state machine */ while (doAnotherStage) { - switch(dctxPtr->dStage) + switch(dctx->dStage) { - case dstage_getHeader: + case dstage_getFrameHeader: if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */ - LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); + size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr); /* will update dStage appropriately */ if (LZ4F_isError(hSize)) return hSize; srcPtr += hSize; break; } - dctxPtr->tmpInSize = 0; - dctxPtr->tmpInTarget = minFHSize; /* minimum to attempt decode */ - dctxPtr->dStage = dstage_storeHeader; - /* pass-through */ - - case dstage_storeHeader: - { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - dctxPtr->tmpInSize += sizeToCopy; + dctx->tmpInSize = 0; + if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */ + dctx->tmpInTarget = minFHSize; /* minimum to attempt decode */ + dctx->dStage = dstage_storeFrameHeader; + /* fall-through */ + + case dstage_storeFrameHeader: + { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr)); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { - nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ - doAnotherStage = 0; /* not enough src data, ask for some more */ - break; - } - { LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); - if (LZ4F_isError(hSize)) return hSize; - } + } + if (dctx->tmpInSize < dctx->tmpInTarget) { + nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ + doAnotherStage = 0; /* not enough src data, ask for some more */ break; } + { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */ + if (LZ4F_isError(hSize)) return hSize; + } + break; - case dstage_getCBlockSize: + case dstage_init: + if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0); + /* internal buffers allocation */ + { size_t const bufferNeeded = dctx->maxBlockSize + + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB); + if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ + dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ + FREEMEM(dctx->tmpIn); + dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize + 4 /* block checksum */); + if (dctx->tmpIn == NULL) + return err0r(LZ4F_ERROR_allocation_failed); + FREEMEM(dctx->tmpOutBuffer); + dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded); + if (dctx->tmpOutBuffer== NULL) + return err0r(LZ4F_ERROR_allocation_failed); + dctx->maxBufferSize = bufferNeeded; + } } + dctx->tmpInSize = 0; + dctx->tmpInTarget = 0; + dctx->tmpOut = dctx->tmpOutBuffer; + dctx->tmpOutStart = 0; + dctx->tmpOutSize = 0; + + dctx->dStage = dstage_getBlockHeader; + /* fall-through */ + + case dstage_getBlockHeader: if ((size_t)(srcEnd - srcPtr) >= BHSize) { selectedIn = srcPtr; srcPtr += BHSize; } else { /* not enough input to read cBlockSize field */ - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlockSize; + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeBlockHeader; } - if (dctxPtr->dStage == dstage_storeCBlockSize) /* can be skipped */ - case dstage_storeCBlockSize: - { size_t sizeToCopy = BHSize - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */ + case dstage_storeBlockHeader: + { size_t const remainingInput = (size_t)(srcEnd - srcPtr); + size_t const wantedData = BHSize - dctx->tmpInSize; + size_t const sizeToCopy = MIN(wantedData, remainingInput); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < BHSize) { /* not enough input to get full cBlockSize; wait for more */ - nextSrcSizeHint = BHSize - dctxPtr->tmpInSize; + dctx->tmpInSize += sizeToCopy; + + if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */ + nextSrcSizeHint = BHSize - dctx->tmpInSize; doAnotherStage = 0; break; } - selectedIn = dctxPtr->tmpIn; - } + selectedIn = dctx->tmpIn; + } /* if (dctx->dStage == dstage_storeBlockHeader) */ - /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */ + /* decode block header */ { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; - if (nextCBlockSize==0) { /* frameEnd signal, no more CBlock */ - dctxPtr->dStage = dstage_getSuffix; + size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4; + if (nextCBlockSize==0) { /* frameEnd signal, no more block */ + dctx->dStage = dstage_getSuffix; break; } - if (nextCBlockSize > dctxPtr->maxBlockSize) return err0r(LZ4F_ERROR_GENERIC); /* invalid cBlockSize */ - dctxPtr->tmpInTarget = nextCBlockSize; + if (nextCBlockSize > dctx->maxBlockSize) + return err0r(LZ4F_ERROR_maxBlockSize_invalid); if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) { - dctxPtr->dStage = dstage_copyDirect; + /* next block is uncompressed */ + dctx->tmpInTarget = nextCBlockSize; + if (dctx->frameInfo.blockChecksumFlag) { + XXH32_reset(&dctx->blockChecksum, 0); + } + dctx->dStage = dstage_copyDirect; break; } - dctxPtr->dStage = dstage_getCBlock; + /* next block is a compressed block */ + dctx->tmpInTarget = nextCBlockSize + crcSize; + dctx->dStage = dstage_getCBlock; if (dstPtr==dstEnd) { - nextSrcSizeHint = nextCBlockSize + BHSize; + nextSrcSizeHint = nextCBlockSize + crcSize + BHSize; doAnotherStage = 0; } break; } case dstage_copyDirect: /* uncompressed block */ - { size_t sizeToCopy = dctxPtr->tmpInTarget; - if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */ - if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; + { size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr)); + size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); memcpy(dstPtr, srcPtr, sizeToCopy); - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy; + if (dctx->frameInfo.blockChecksumFlag) { + XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + } + if (dctx->frameInfo.contentChecksumFlag) + XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= sizeToCopy; - /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); + /* history management (linked blocks only)*/ + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) + LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0); srcPtr += sizeToCopy; dstPtr += sizeToCopy; - if (sizeToCopy == dctxPtr->tmpInTarget) { /* all copied */ - dctxPtr->dStage = dstage_getCBlockSize; + if (sizeToCopy == dctx->tmpInTarget) { /* all done */ + if (dctx->frameInfo.blockChecksumFlag) { + dctx->tmpInSize = 0; + dctx->dStage = dstage_getBlockChecksum; + } else + dctx->dStage = dstage_getBlockHeader; /* new block */ break; } - dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ - nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize; + dctx->tmpInTarget -= sizeToCopy; /* need to copy more */ + nextSrcSizeHint = dctx->tmpInTarget + + + dctx->frameInfo.contentChecksumFlag * 4 /* block checksum */ + + BHSize /* next header size */; doAnotherStage = 0; break; } - case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */ - if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) { - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlock; + /* check block checksum for recently transferred uncompressed block */ + case dstage_getBlockChecksum: + { const void* crcSrc; + if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) { + crcSrc = srcPtr; + srcPtr += 4; + } else { + size_t const stillToCopy = 4 - dctx->tmpInSize; + size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr)); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctx->tmpInSize < 4) { /* all input consumed */ + doAnotherStage = 0; + break; + } + crcSrc = dctx->header; + } + { U32 const readCRC = LZ4F_readLE32(crcSrc); + U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); + if (readCRC != calcCRC) + return err0r(LZ4F_ERROR_blockChecksum_invalid); + } + } + dctx->dStage = dstage_getBlockHeader; /* new block */ + break; + + case dstage_getCBlock: + if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) { + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeCBlock; break; } + /* input large enough to read full block directly */ selectedIn = srcPtr; - srcPtr += dctxPtr->tmpInTarget; - dctxPtr->dStage = dstage_decodeCBlock; + srcPtr += dctx->tmpInTarget; + dctx->dStage = dstage_decodeCBlock; break; case dstage_storeCBlock: - { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - dctxPtr->tmpInSize += sizeToCopy; + { size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize; + size_t const inputLeft = (size_t)(srcEnd-srcPtr); + size_t const sizeToCopy = MIN(wantedData, inputLeft); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* need more input */ - nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; + if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */ + nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; doAnotherStage=0; break; } - selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_decodeCBlock; - /* pass-through */ + selectedIn = dctx->tmpIn; + dctx->dStage = dstage_decodeCBlock; } + /* fall-through */ + /* At this stage, input is large enough to decode a block */ case dstage_decodeCBlock: - if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */ - dctxPtr->dStage = dstage_decodeCBlock_intoTmp; + if (dctx->frameInfo.blockChecksumFlag) { + dctx->tmpInTarget -= 4; + { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget); + U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0); + if (readBlockCrc != calcBlockCrc) + return err0r(LZ4F_ERROR_blockChecksum_invalid); + } } + if ((size_t)(dstEnd-dstPtr) < dctx->maxBlockSize) /* not enough place into dst : decode into tmpOut */ + dctx->dStage = dstage_decodeCBlock_intoTmp; else - dctxPtr->dStage = dstage_decodeCBlock_intoDst; + dctx->dStage = dstage_decodeCBlock_intoDst; break; case dstage_decodeCBlock_intoDst: - { int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; - - if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) - decoder = LZ4_decompress_safe_usingDict; - else - decoder = LZ4F_decompress_safe; - - decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + { int const decodedSize = LZ4_decompress_safe_usingDict( + (const char*)selectedIn, (char*)dstPtr, + (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, + (const char*)dctx->dict, (int)dctx->dictSize); if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; + if (dctx->frameInfo.contentChecksumFlag) + XXH32_update(&(dctx->xxh), dstPtr, decodedSize); + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= decodedSize; /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); + if (dctx->frameInfo.blockMode==LZ4F_blockLinked) + LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0); dstPtr += decodedSize; - dctxPtr->dStage = dstage_getCBlockSize; + dctx->dStage = dstage_getBlockHeader; break; } case dstage_decodeCBlock_intoTmp: /* not enough place into dst : decode into tmpOut */ - { int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; - - if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) - decoder = LZ4_decompress_safe_usingDict; - else - decoder = LZ4F_decompress_safe; - - /* ensure enough place for tmpOut */ - if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) { - if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { - if (dctxPtr->dictSize > 128 KB) { - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB); - dctxPtr->dictSize = 64 KB; - } - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize; - } else { /* dict not within tmp */ - size_t reservedDictSpace = dctxPtr->dictSize; - if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; + /* ensure enough place for tmpOut */ + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) { + if (dctx->dict == dctx->tmpOutBuffer) { + if (dctx->dictSize > 128 KB) { + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB); + dctx->dictSize = 64 KB; } + dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize; + } else { /* dict not within tmp */ + size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB); + dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace; } + } - /* Decode */ - decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return err0r(LZ4F_ERROR_decompressionFailed); /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; - dctxPtr->tmpOutSize = decodedSize; - dctxPtr->tmpOutStart = 0; - dctxPtr->dStage = dstage_flushOut; - break; + /* Decode block */ + { int const decodedSize = LZ4_decompress_safe_usingDict( + (const char*)selectedIn, (char*)dctx->tmpOut, + (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, + (const char*)dctx->dict, (int)dctx->dictSize); + if (decodedSize < 0) /* decompression failed */ + return err0r(LZ4F_ERROR_decompressionFailed); + if (dctx->frameInfo.contentChecksumFlag) + XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize); + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= decodedSize; + dctx->tmpOutSize = decodedSize; + dctx->tmpOutStart = 0; + dctx->dStage = dstage_flushOut; } + /* fall-through */ case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ - { size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart; - if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr; - memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); + { size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr)); + memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy); /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); + if (dctx->frameInfo.blockMode==LZ4F_blockLinked) + LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1); - dctxPtr->tmpOutStart += sizeToCopy; + dctx->tmpOutStart += sizeToCopy; dstPtr += sizeToCopy; - /* end of flush ? */ - if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) { - dctxPtr->dStage = dstage_getCBlockSize; + if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */ + dctx->dStage = dstage_getBlockHeader; /* get next block */ break; } nextSrcSizeHint = BHSize; @@ -1277,45 +1524,45 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, } case dstage_getSuffix: - { size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; - if (dctxPtr->frameRemainingSize) return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */ - if (suffixSize == 0) { /* frame completed */ - nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; - doAnotherStage = 0; - break; - } - if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */ - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeSuffix; - } else { - selectedIn = srcPtr; - srcPtr += 4; - } + if (dctx->frameRemainingSize) + return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */ + if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */ + nextSrcSizeHint = 0; + LZ4F_resetDecompressionContext(dctx); + doAnotherStage = 0; + break; + } + if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */ + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeSuffix; + } else { + selectedIn = srcPtr; + srcPtr += 4; } - if (dctxPtr->dStage == dstage_storeSuffix) /* can be skipped */ + if (dctx->dStage == dstage_storeSuffix) /* can be skipped */ case dstage_storeSuffix: - { - size_t sizeToCopy = 4 - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + { size_t const remainingInput = (size_t)(srcEnd - srcPtr); + size_t const wantedData = 4 - dctx->tmpInSize; + size_t const sizeToCopy = MIN(wantedData, remainingInput); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < 4) { /* not enough input to read complete suffix */ - nextSrcSizeHint = 4 - dctxPtr->tmpInSize; + dctx->tmpInSize += sizeToCopy; + if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */ + nextSrcSizeHint = 4 - dctx->tmpInSize; doAnotherStage=0; break; } - selectedIn = dctxPtr->tmpIn; - } + selectedIn = dctx->tmpIn; + } /* if (dctx->dStage == dstage_storeSuffix) */ - /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */ + /* case dstage_checkSuffix: */ /* no direct call, avoid scan-build warning */ { U32 const readCRC = LZ4F_readLE32(selectedIn); - U32 const resultCRC = XXH32_digest(&(dctxPtr->xxh)); - if (readCRC != resultCRC) return err0r(LZ4F_ERROR_contentChecksum_invalid); + U32 const resultCRC = XXH32_digest(&(dctx->xxh)); + if (readCRC != resultCRC) + return err0r(LZ4F_ERROR_contentChecksum_invalid); nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; + LZ4F_resetDecompressionContext(dctx); doAnotherStage = 0; break; } @@ -1326,77 +1573,79 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, srcPtr += 4; } else { /* not enough input to read cBlockSize field */ - dctxPtr->tmpInSize = 4; - dctxPtr->tmpInTarget = 8; - dctxPtr->dStage = dstage_storeSFrameSize; + dctx->tmpInSize = 4; + dctx->tmpInTarget = 8; + dctx->dStage = dstage_storeSFrameSize; } - if (dctxPtr->dStage == dstage_storeSFrameSize) + if (dctx->dStage == dstage_storeSFrameSize) case dstage_storeSFrameSize: { - size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, + (size_t)(srcEnd - srcPtr) ); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* not enough input to get full sBlockSize; wait for more */ - nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + dctx->tmpInSize += sizeToCopy; + if (dctx->tmpInSize < dctx->tmpInTarget) { + /* not enough input to get full sBlockSize; wait for more */ + nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize; doAnotherStage = 0; break; } - selectedIn = dctxPtr->header + 4; - } + selectedIn = dctx->header + 4; + } /* if (dctx->dStage == dstage_storeSFrameSize) */ /* case dstage_decodeSFrameSize: */ /* no direct access */ { size_t const SFrameSize = LZ4F_readLE32(selectedIn); - dctxPtr->frameInfo.contentSize = SFrameSize; - dctxPtr->tmpInTarget = SFrameSize; - dctxPtr->dStage = dstage_skipSkippable; + dctx->frameInfo.contentSize = SFrameSize; + dctx->tmpInTarget = SFrameSize; + dctx->dStage = dstage_skipSkippable; break; } case dstage_skipSkippable: - { size_t skipSize = dctxPtr->tmpInTarget; - if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr; + { size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr)); srcPtr += skipSize; - dctxPtr->tmpInTarget -= skipSize; + dctx->tmpInTarget -= skipSize; doAnotherStage = 0; - nextSrcSizeHint = dctxPtr->tmpInTarget; - if (nextSrcSizeHint) break; - dctxPtr->dStage = dstage_getHeader; + nextSrcSizeHint = dctx->tmpInTarget; + if (nextSrcSizeHint) break; /* still more to skip */ + /* frame fully skipped : prepare context for a new frame */ + LZ4F_resetDecompressionContext(dctx); break; } } - } - - /* preserve dictionary within tmp if necessary */ - if ( (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) - &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) - &&(!decompressOptionsPtr->stableDst) - &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) - ) + } /* while (doAnotherStage) */ + + /* preserve history within tmp whenever necessary */ + LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2); + if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */ + && (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */ + && (!decompressOptionsPtr->stableDst) /* cannot rely on dst data to remain there for next call */ + && ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */ { - if (dctxPtr->dStage == dstage_flushOut) { - size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - size_t copySize = 64 KB - dctxPtr->tmpOutSize; - const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; - if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; + if (dctx->dStage == dstage_flushOut) { + size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer; + size_t copySize = 64 KB - dctx->tmpOutSize; + const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; + if (dctx->tmpOutSize > 64 KB) copySize = 0; if (copySize > preserveSize) copySize = preserveSize; - memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + if (copySize > 0) + memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart; + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dctx->tmpOutStart; } else { - size_t newDictSize = dctxPtr->dictSize; - const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; - if ((newDictSize) > 64 KB) newDictSize = 64 KB; + const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize; + size_t const newDictSize = MIN(dctx->dictSize, 64 KB); - memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); + if (newDictSize > 0) + memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = newDictSize; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize; + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = newDictSize; + dctx->tmpOut = dctx->tmpOutBuffer + newDictSize; } } @@ -1404,3 +1653,23 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, *dstSizePtr = (dstPtr - dstStart); return nextSrcSizeHint; } + +/*! LZ4F_decompress_usingDict() : + * Same as LZ4F_decompress(), using a predefined dictionary. + * Dictionary is used "in place", without any preprocessing. + * It must remain accessible throughout the entire frame decoding. + */ +size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr) +{ + if (dctx->dStage <= dstage_init) { + dctx->dict = (const BYTE*)dict; + dctx->dictSize = dictSize; + } + return LZ4F_decompress(dctx, dstBuffer, dstSizePtr, + srcBuffer, srcSizePtr, + decompressOptionsPtr); +} diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 3104d2e..675aeeb 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -1,7 +1,7 @@ /* LZ4 auto-framing library Header File - Copyright (C) 2011-2016, Yann Collet. + Copyright (C) 2011-2017, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -48,49 +48,68 @@ extern "C" { /* --- Dependency --- */ #include <stddef.h> /* size_t */ -/*-*************************************************************** -* Compiler specifics -*****************************************************************/ -/*! -* LZ4_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL + +/** + Introduction + + lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md). + lz4frame.h provides frame compression functions that take care + of encoding standard metadata alongside LZ4-compressed blocks. */ + +/*-*************************************************************** + * Compiler specifics + *****************************************************************/ +/* LZ4_DLL_EXPORT : + * Enable exporting of functions when building a Windows DLL + * LZ4FLIB_API : + * Control library symbols visibility. + */ #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) # define LZ4FLIB_API __declspec(dllexport) #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) # define LZ4FLIB_API __declspec(dllimport) +#elif defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4FLIB_API __attribute__ ((__visibility__ ("default"))) #else # define LZ4FLIB_API #endif -#if defined(_MSC_VER) -# define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */ -#elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6)) -# define LZ4F_DEPRECATE(x) x __attribute__((deprecated)) +#ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS +# define LZ4F_DEPRECATE(x) x #else -# define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */ +# if defined(_MSC_VER) +# define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */ +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6)) +# define LZ4F_DEPRECATE(x) x __attribute__((deprecated)) +# else +# define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */ +# endif #endif /*-************************************ -* Error management -**************************************/ + * Error management + **************************************/ typedef size_t LZ4F_errorCode_t; -LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); -LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return error code string; useful for debugging */ +LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells if a `LZ4F_errorCode_t` function result is an error code */ +LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; useful for debugging */ /*-************************************ -* Frame compression types -**************************************/ -/* #define LZ4F_DISABLE_OBSOLETE_ENUMS */ /* uncomment to disable obsolete enums */ -#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS + * Frame compression types + **************************************/ +/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */ +#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS # define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x #else # define LZ4F_OBSOLETE_ENUM(x) #endif +/* The larger the block size, the (slightly) better the compression ratio, + * though there are diminishing returns. + * Larger blocks also increase memory usage on both compression and decompression sides. */ typedef enum { LZ4F_default=0, LZ4F_max64KB=4, @@ -103,6 +122,9 @@ typedef enum { LZ4F_OBSOLETE_ENUM(max4MB) } LZ4F_blockSizeID_t; +/* Linked blocks sharply reduce inefficiencies when using small blocks, + * they compress better. + * However, some LZ4 decoders are only compatible with independent blocks */ typedef enum { LZ4F_blockLinked=0, LZ4F_blockIndependent @@ -118,63 +140,70 @@ typedef enum { } LZ4F_contentChecksum_t; typedef enum { + LZ4F_noBlockChecksum=0, + LZ4F_blockChecksumEnabled +} LZ4F_blockChecksum_t; + +typedef enum { LZ4F_frame=0, LZ4F_skippableFrame LZ4F_OBSOLETE_ENUM(skippableFrame) } LZ4F_frameType_t; -#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS +#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS typedef LZ4F_blockSizeID_t blockSizeID_t; typedef LZ4F_blockMode_t blockMode_t; typedef LZ4F_frameType_t frameType_t; typedef LZ4F_contentChecksum_t contentChecksum_t; #endif -/* LZ4F_frameInfo_t : - * makes it possible to supply detailed frame parameters to the stream interface. - * It's not required to set all fields, as long as the structure was initially memset() to zero. - * All reserved fields must be set to zero. */ +/*! LZ4F_frameInfo_t : + * makes it possible to set or read frame parameters. + * It's not required to set all fields, as long as the structure was initially memset() to zero. + * For all fields, 0 sets it to default value */ typedef struct { - LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ - LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ - LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */ - unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */ - unsigned reserved[2]; /* must be zero for forward compatibility */ + LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */ + LZ4F_contentChecksum_t contentChecksumFlag; /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */ + LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */ + unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */ + unsigned dictID; /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */ + LZ4F_blockChecksum_t blockChecksumFlag; /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */ } LZ4F_frameInfo_t; -/* LZ4F_preferences_t : - * makes it possible to supply detailed compression parameters to the stream interface. - * It's not required to set all fields, as long as the structure was initially memset() to zero. - * All reserved fields must be set to zero. */ +/*! LZ4F_preferences_t : + * makes it possible to supply detailed compression parameters to the stream interface. + * It's not required to set all fields, as long as the structure was initially memset() to zero. + * All reserved fields must be set to zero. */ typedef struct { LZ4F_frameInfo_t frameInfo; - int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */ - unsigned autoFlush; /* 1 == always flush (reduce usage of tmp buffer) */ + int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */ + unsigned autoFlush; /* 1 == always flush, to reduce usage of internal buffers */ unsigned reserved[4]; /* must be zero for forward compatibility */ } LZ4F_preferences_t; +LZ4FLIB_API int LZ4F_compressionLevel_max(void); + /*-********************************* * Simple compression function ***********************************/ -/*!LZ4F_compressFrameBound() : - * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. - * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. +/*! LZ4F_compressFrameBound() : + * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. + * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. */ LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); -/*!LZ4F_compressFrame() : - * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1 - * An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation. - * This value is supplied by LZ4F_compressFrameBound(). - * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode). - * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. +/*! LZ4F_compressFrame() : + * Compress an entire srcBuffer into a valid LZ4 frame. + * dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. * @return : number of bytes written into dstBuffer. * or an error code if it fails (can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr); - +LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr); /*-*********************************** @@ -188,55 +217,57 @@ typedef struct { unsigned reserved[3]; } LZ4F_compressOptions_t; -/* Resource Management */ +/*--- Resource Management ---*/ #define LZ4F_VERSION 100 LZ4FLIB_API unsigned LZ4F_getVersion(void); -LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); -LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); -/* LZ4F_createCompressionContext() : +/*! LZ4F_createCompressionContext() : * The first thing to do is to create a compressionContext object, which will be used in all compression operations. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version. * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. * The function will provide a pointer to a fully allocated LZ4F_cctx object. * If @return != zero, there was an error during context creation. * Object can release its memory using LZ4F_freeCompressionContext(); */ +LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); +LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); -/* Compression */ +/*---- Compression ----*/ -LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr); -/* LZ4F_compressBegin() : - * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header. Maximum header size is 15 bytes. +#define LZ4F_HEADER_SIZE_MAX 19 +/*! LZ4F_compressBegin() : + * will write the frame header into dstBuffer. + * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. * @return : number of bytes written into dstBuffer for the header * or an error code (which can be tested using LZ4F_isError()) */ - -LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); -/* LZ4F_compressBound() : - * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. - * Different preferences can produce different results. - * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. - * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) +LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* prefsPtr); + +/*! LZ4F_compressBound() : + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. */ +LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); -LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); -/* LZ4F_compressUpdate() : +/*! LZ4F_compressUpdate() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * An important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. - * This value is provided by using LZ4F_compressBound(). + * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. + * This value is provided by LZ4F_compressBound(). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). * LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) */ +LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); -LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); -/* LZ4F_flush() : +/*! LZ4F_flush() : * When data must be generated and sent immediately, without waiting for a block to be completely filled, * it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. * `dstCapacity` must be large enough to ensure the operation will be successful. @@ -244,17 +275,18 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapaci * @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx) * or an error code if it fails (which can be tested using LZ4F_isError()) */ +LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); -LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr); -/* LZ4F_compressEnd() : - * To properly finish the compressed frame, invoke LZ4F_compressEnd(). - * It will flush whatever data remained within `cctx` (like LZ4_flush()) - * and properly finalize the frame, with an endMark and a checksum. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. +/*! LZ4F_compressEnd() : + * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). + * It will flush whatever data remained within `cctx` (like LZ4_flush()) + * and properly finalize the frame, with an endMark and a checksum. + * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) * or an error code if it fails (which can be tested using LZ4F_isError()) - * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. + * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. */ +LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); /*-********************************* @@ -264,66 +296,83 @@ typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */ typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */ typedef struct { - unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */ - unsigned reserved[3]; + unsigned stableDst; /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */ + unsigned reserved[3]; /* must be set to zero for forward compatibility */ } LZ4F_decompressOptions_t; /* Resource management */ /*!LZ4F_createDecompressionContext() : - * Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations. - * The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions. - * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object. + * Create an LZ4F_dctx object, to track all decompression operations. + * The version provided MUST be LZ4F_VERSION. + * The function provides a pointer to an allocated and initialized LZ4F_dctx object. * The result is an errorCode, which can be tested using LZ4F_isError(). * dctx memory can be released using LZ4F_freeDecompressionContext(); * The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. * That is, it should be == 0 if decompression has been completed fully and correctly. */ LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version); -LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx); +LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); -/*====== Decompression ======*/ +/*-*********************************** +* Streaming decompression functions +*************************************/ -/*!LZ4F_getFrameInfo() : - * This function decodes frame header information (such as max blockSize, frame checksum, etc.). - * Its usage is optional. The objective is to extract frame header information, typically for allocation purposes. - * A header size is variable and can length from 7 to 15 bytes. It's possible to provide more input bytes than that. - * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - * Decompression must resume from this point (srcBuffer + *srcSizePtr). - * Note that LZ4F_getFrameInfo() can also be used anytime *after* decompression is started, in which case 0 input byte can be enough. - * Frame header info is *copied into* an already allocated LZ4F_frameInfo_t structure. +/*! LZ4F_getFrameInfo() : + * This function extracts frame parameters (max blockSize, dictID, etc.). + * Its usage is optional. + * Extracted information is typically useful for allocation and dictionary. + * This function works in 2 situations : + * - At the beginning of a new frame, in which case + * it will decode information from `srcBuffer`, starting the decoding process. + * Input size must be large enough to successfully decode the entire frame header. + * Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. + * It's allowed to provide more input data than this minimum. + * - After decoding has been started. + * In which case, no input is read, frame parameters are extracted from dctx. + * - If decoding has barely started, but not yet extracted information from header, + * LZ4F_getFrameInfo() will fail. + * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + * Decompression must resume from (srcBuffer + *srcSizePtr). * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, - * or an error code which can be tested using LZ4F_isError() - * (typically, when there is not enough src bytes to fully decode the frame header) + * or an error code which can be tested using LZ4F_isError(). + * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. + * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. */ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr); -/*!LZ4F_decompress() : - * Call this function repetitively to regenerate data compressed within `srcBuffer`. - * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. +/*! LZ4F_decompress() : + * Call this function repetitively to regenerate compressed data from `srcBuffer`. + * The function will read up to *srcSizePtr bytes from srcBuffer, + * and decompress data into dstBuffer, of capacity *dstSizePtr. + * + * The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value). + * The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value). * - * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + * The function does not necessarily read all input bytes, so always check value in *srcSizePtr. + * Unconsumed source data must be presented again in subsequent invocations. * - * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). - * Number of bytes read can be < number of bytes provided, meaning there is some more data to decode. - * It typically happens when dstBuffer is not large enough to contain all decoded data. - * Remaining data will have to be presented again in a subsequent invocation. + * `dstBuffer` can freely change between each consecutive function invocation. + * `dstBuffer` content will be overwritten. * - * `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten. - * `dstBuffer` can be changed at will between each consecutive function invocation. + * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides some small speed benefit, because it skips intermediate buffers. + * This is just a hint though, it's always possible to provide any srcSize. * - * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. - * Schematically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some boost to performance, since it does skip intermediate buffers. - * This is just a hint though, it's always possible to provide any srcSize. - * When a frame is fully decoded, @return will be 0 (no more data expected). - * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). + * When a frame is fully decoded, @return will be 0 (no more data expected). + * When provided with more bytes than necessary to decode a frame, + * LZ4F_decompress() will stop reading exactly at end of current frame, and @return 0. * - * After a frame is fully decoded, dctx can be used again to decompress another frame. + * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). + * After a decompression error, the `dctx` context is not resumable. + * Use LZ4F_resetDecompressionContext() to return to clean state. + * + * After a frame is fully decoded, dctx can be used again to decompress another frame. */ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx, void* dstBuffer, size_t* dstSizePtr, @@ -331,6 +380,14 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx, const LZ4F_decompressOptions_t* dOptPtr); +/*! LZ4F_resetDecompressionContext() : added in v1.8.0 + * In case of an error, the context is left in "undefined" state. + * In which case, it's necessary to reset it, before re-using it. + * This method can also be used to abruptly stop any unfinished decompression, + * and start a new one using same context resources. */ +LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */ + + #if defined (__cplusplus) } diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index f2228a5..a59b94b 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -43,7 +43,15 @@ extern "C" { /* lz4frame_static.h should be used solely in the context of static linking. * It contains definitions which are not stable and may change in the future. * Never use it in the context of DLL linking. - * */ + * + * Defining LZ4F_PUBLISH_STATIC_FUNCTIONS allows one to override this. Use at + * your own risk. + */ +#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS +#define LZ4FLIB_STATIC_API LZ4FLIB_API +#else +#define LZ4FLIB_STATIC_API +#endif /* --- Dependency --- */ @@ -52,27 +60,91 @@ extern "C" { /* --- Error List --- */ #define LZ4F_LIST_ERRORS(ITEM) \ - ITEM(OK_NoError) ITEM(ERROR_GENERIC) \ - ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \ + ITEM(OK_NoError) \ + ITEM(ERROR_GENERIC) \ + ITEM(ERROR_maxBlockSize_invalid) \ + ITEM(ERROR_blockMode_invalid) \ + ITEM(ERROR_contentChecksumFlag_invalid) \ ITEM(ERROR_compressionLevel_invalid) \ - ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \ + ITEM(ERROR_headerVersion_wrong) \ + ITEM(ERROR_blockChecksum_invalid) \ + ITEM(ERROR_reservedFlag_set) \ ITEM(ERROR_allocation_failed) \ - ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \ - ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \ + ITEM(ERROR_srcSize_tooLarge) \ + ITEM(ERROR_dstMaxSize_tooSmall) \ + ITEM(ERROR_frameHeader_incomplete) \ + ITEM(ERROR_frameType_unknown) \ + ITEM(ERROR_frameSize_wrong) \ ITEM(ERROR_srcPtr_wrong) \ ITEM(ERROR_decompressionFailed) \ - ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \ + ITEM(ERROR_headerChecksum_invalid) \ + ITEM(ERROR_contentChecksum_invalid) \ + ITEM(ERROR_frameDecoding_alreadyStarted) \ ITEM(ERROR_maxCode) -#define LZ4F_DISABLE_OLD_ENUMS /* comment to enable deprecated enums */ -#ifndef LZ4F_DISABLE_OLD_ENUMS -# define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, ENUM = LZ4F_##ENUM, -#else -# define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, -#endif -typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */ - -LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); +#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, + +/* enum list is exposed, to handle specific errors */ +typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; + +LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); + + + +/********************************** + * Bulk processing dictionary API + *********************************/ +typedef struct LZ4F_CDict_s LZ4F_CDict; + +/*! LZ4_createCDict() : + * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */ +LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); +LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict); + + +/*! LZ4_compressFrame_usingCDict() : + * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary. + * If cdict==NULL, compress without a dictionary. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * If this condition is not respected, function will fail (@return an errorCode). + * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, + * but it's not recommended, as it's the only way to provide dictID in the frame header. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) */ +LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict( + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr); + + +/*! LZ4F_compressBegin_usingCDict() : + * Inits streaming dictionary compression, and writes the frame header into dstBuffer. + * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * `prefsPtr` is optional : you may provide NULL as argument, + * however, it's the only way to provide dictID in the frame header. + * @return : number of bytes written into dstBuffer for the header, + * or an error code (which can be tested using LZ4F_isError()) */ +LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict( + LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* prefsPtr); + + +/*! LZ4F_decompress_usingDict() : + * Same as LZ4F_decompress(), using a predefined dictionary. + * Dictionary is used "in place", without any preprocessing. + * It must remain accessible throughout the entire frame decoding. */ +LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict( + LZ4F_dctx* dctxPtr, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr); #if defined (__cplusplus) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 298550c..f2c2566 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -1,6 +1,6 @@ /* LZ4 HC - High Compression Mode of LZ4 - Copyright (C) 2011-2016, Yann Collet. + Copyright (C) 2011-2017, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -38,54 +38,43 @@ * Tuning Parameter ***************************************/ -/*! - * HEAPMODE : - * Select how default compression function will allocate workplace memory, - * in stack (0:fastest), or in heap (1:requires malloc()). - * Since workplace is rather large, heap mode is recommended. +/*! HEAPMODE : + * Select how default compression function will allocate workplace memory, + * in stack (0:fastest), or in heap (1:requires malloc()). + * Since workplace is rather large, heap mode is recommended. */ #ifndef LZ4HC_HEAPMODE # define LZ4HC_HEAPMODE 1 #endif -/* ************************************* -* Dependency -***************************************/ +/*=== Dependency ===*/ +#define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" -/* ************************************* -* Local Compiler Options -***************************************/ +/*=== Common LZ4 definitions ===*/ #if defined(__GNUC__) # pragma GCC diagnostic ignored "-Wunused-function" #endif - #if defined (__clang__) # pragma clang diagnostic ignored "-Wunused-function" #endif - -/* ************************************* -* Common LZ4 definition -***************************************/ #define LZ4_COMMONDEFS_ONLY -#include "lz4.c" +#include "lz4.c" /* LZ4_count, constants, mem */ -/* ************************************* -* Local Constants -***************************************/ +/*=== Constants ===*/ #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) -/************************************** -* Local Macros -**************************************/ -#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) -/* #define DELTANEXTU16(p) chainTable[(p) & LZ4HC_MAXD_MASK] */ /* flexible, LZ4HC_MAXD dependent */ -#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ +/*=== Macros ===*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) +#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) +#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ +#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } @@ -108,7 +97,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) /* Update chains up to ip (excluded) */ -FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) +LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) { U16* const chainTable = hc4->chainTable; U32* const hashTable = hc4->hashTable; @@ -120,7 +109,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) U32 const h = LZ4HC_hashPtr(base+idx); size_t delta = idx - hashTable[h]; if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; - DELTANEXTU16(idx) = (U16)delta; + DELTANEXTU16(chainTable, idx) = (U16)delta; hashTable[h] = idx; idx++; } @@ -128,56 +117,73 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) hc4->nextToUpdate = target; } - -FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iLimit, - const BYTE** matchpos, - const int maxNbAttempts) +/** LZ4HC_countBack() : + * @return : negative value, nb of common bytes before ip/match */ +LZ4_FORCE_INLINE +int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, + const BYTE* const iMin, const BYTE* const mMin) { - U16* const chainTable = hc4->chainTable; - U32* const HashTable = hc4->hashTable; - const BYTE* const base = hc4->base; - const BYTE* const dictBase = hc4->dictBase; - const U32 dictLimit = hc4->dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); - U32 matchIndex; - int nbAttempts=maxNbAttempts; - size_t ml=0; + int back=0; + while ( (ip+back > iMin) + && (match+back > mMin) + && (ip[back-1] == match[back-1])) + back--; + return back; +} - /* HC4 match finder */ - LZ4HC_Insert(hc4, ip); - matchIndex = HashTable[LZ4HC_hashPtr(ip)]; +/* LZ4HC_countPattern() : + * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */ +static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32) +{ + const BYTE* const iStart = ip; + reg_t const pattern = (sizeof(pattern)==8) ? (reg_t)pattern32 + (((reg_t)pattern32) << 32) : pattern32; + + while (likely(ip < iEnd-(sizeof(pattern)-1))) { + reg_t const diff = LZ4_read_ARCH(ip) ^ pattern; + if (!diff) { ip+=sizeof(pattern); continue; } + ip += LZ4_NbCommonBytes(diff); + return (unsigned)(ip - iStart); + } - while ((matchIndex>=lowLimit) && (nbAttempts)) { - nbAttempts--; - if (matchIndex >= dictLimit) { - const BYTE* const match = base + matchIndex; - if (*(match+ml) == *(ip+ml) - && (LZ4_read32(match) == LZ4_read32(ip))) - { - size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if (mlt > ml) { ml = mlt; *matchpos = match; } - } - } else { - const BYTE* const match = dictBase + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) { - size_t mlt; - const BYTE* vLimit = ip + (dictLimit - matchIndex); - if (vLimit > iLimit) vLimit = iLimit; - mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; - if ((ip+mlt == vLimit) && (vLimit < iLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit); - if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ - } + if (LZ4_isLittleEndian()) { + reg_t patternByte = pattern; + while ((ip<iEnd) && (*ip == (BYTE)patternByte)) { + ip++; patternByte >>= 8; + } + } else { /* big endian */ + U32 bitOffset = (sizeof(pattern)*8) - 8; + while (ip < iEnd) { + BYTE const byte = (BYTE)(pattern >> bitOffset); + if (*ip != byte) break; + ip ++; bitOffset -= 8; } - matchIndex -= DELTANEXTU16(matchIndex); } - return (int)ml; + return (unsigned)(ip - iStart); } +/* LZ4HC_reverseCountPattern() : + * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) + * read using natural platform endianess */ +static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) +{ + const BYTE* const iStart = ip; -FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( + while (likely(ip >= iLow+4)) { + if (LZ4_read32(ip-4) != pattern) break; + ip -= 4; + } + { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianess */ + while (likely(ip>iLow)) { + if (ip[-1] != *bytePtr) break; + ip--; bytePtr--; + } } + return (unsigned)(iStart - ip); +} + +typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; + +LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( LZ4HC_CCtx_internal* hc4, const BYTE* const ip, const BYTE* const iLowLimit, @@ -185,98 +191,176 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( int longest, const BYTE** matchpos, const BYTE** startpos, - const int maxNbAttempts) + const int maxNbAttempts, + const int patternAnalysis) { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; const BYTE* const base = hc4->base; const U32 dictLimit = hc4->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); + const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - MAX_DISTANCE; const BYTE* const dictBase = hc4->dictBase; - U32 matchIndex; + int const delta = (int)(ip-iLowLimit); int nbAttempts = maxNbAttempts; - int delta = (int)(ip-iLowLimit); - + U32 const pattern = LZ4_read32(ip); + U32 matchIndex; + repeat_state_e repeat = rep_untested; + size_t srcPatternLength = 0; + DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch"); /* First Match */ LZ4HC_Insert(hc4, ip); matchIndex = HashTable[LZ4HC_hashPtr(ip)]; + DEBUGLOG(7, "First match at index %u / %u (lowLimit)", + matchIndex, lowLimit); while ((matchIndex>=lowLimit) && (nbAttempts)) { + DEBUGLOG(7, "remaining attempts : %i", nbAttempts); nbAttempts--; if (matchIndex >= dictLimit) { - const BYTE* matchPtr = base + matchIndex; + const BYTE* const matchPtr = base + matchIndex; if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) { - if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { + if (LZ4_read32(matchPtr) == pattern) { int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); + #if 0 + /* more generic but unfortunately slower on clang */ + int const back = LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr); + #else int back = 0; - - while ((ip+back > iLowLimit) - && (matchPtr+back > lowPrefixPtr) - && (ip[back-1] == matchPtr[back-1])) + while ( (ip+back > iLowLimit) + && (matchPtr+back > lowPrefixPtr) + && (ip[back-1] == matchPtr[back-1])) { back--; - + } + #endif mlt -= back; if (mlt > longest) { - longest = (int)mlt; + longest = mlt; *matchpos = matchPtr+back; *startpos = ip+back; - } - } + } } } - } else { + } else { /* matchIndex < dictLimit */ const BYTE* const matchPtr = dictBase + matchIndex; - if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { - size_t mlt; - int back=0; + if (LZ4_read32(matchPtr) == pattern) { + int mlt; + int back = 0; const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--; + while ( (ip+back > iLowLimit) + && (matchIndex+back > lowLimit) + && (ip[back-1] == matchPtr[back-1])) + back--; mlt -= back; - if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } - } - } - matchIndex -= DELTANEXTU16(matchIndex); - } + if (mlt > longest) { + longest = mlt; + *matchpos = base + matchIndex + back; + *startpos = ip + back; + } } } + + { U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex); + matchIndex -= nextOffset; + if (patternAnalysis && nextOffset==1) { + /* may be a repeated pattern */ + if (repeat == rep_untested) { + if ( ((pattern & 0xFFFF) == (pattern >> 16)) + & ((pattern & 0xFF) == (pattern >> 24)) ) { + repeat = rep_confirmed; + srcPatternLength = LZ4HC_countPattern(ip+4, iHighLimit, pattern) + 4; + } else { + repeat = rep_not; + } } + if ( (repeat == rep_confirmed) + && (matchIndex >= dictLimit) ) { /* same segment only */ + const BYTE* const matchPtr = base + matchIndex; + if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ + size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); + const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE; + size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern); + size_t const currentSegmentLength = backLength + forwardPatternLength; + + if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ + && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ + matchIndex += (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ + } else { + matchIndex -= (U32)backLength; /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */ + } + } } } } + } /* while ((matchIndex>=lowLimit) && (nbAttempts)) */ return longest; } +LZ4_FORCE_INLINE +int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, + const BYTE** matchpos, + const int maxNbAttempts, + const int patternAnalysis) +{ + const BYTE* uselessPtr = ip; + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis); +} + -typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; -#define LZ4HC_DEBUG 0 -#if LZ4HC_DEBUG -static unsigned debug = 0; -#endif +typedef enum { + noLimit = 0, + limitedOutput = 1, + limitedDestSize = 2, +} limitedOutput_directive; -FORCE_INLINE int LZ4HC_encodeSequence ( +/* LZ4HC_encodeSequence() : + * @return : 0 if ok, + * 1 if buffer issue detected */ +LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( const BYTE** ip, BYTE** op, const BYTE** anchor, int matchLength, const BYTE* const match, - limitedOutput_directive limitedOutputBuffer, + limitedOutput_directive limit, BYTE* oend) { - int length; - BYTE* token; - -#if LZ4HC_DEBUG - if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); + size_t length; + BYTE* const token = (*op)++; + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2) + static const BYTE* start = NULL; + static U32 totalCost = 0; + U32 const pos = (start==NULL) ? 0 : (U32)(*anchor - start); + U32 const ll = (U32)(*ip - *anchor); + U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0; + U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0; + U32 const cost = 1 + llAdd + ll + 2 + mlAdd; + if (start==NULL) start = *anchor; /* only works for single segment */ + //g_debuglog_enable = (pos >= 2228) & (pos <= 2262); + DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u", + pos, + (U32)(*ip - *anchor), matchLength, (U32)(*ip-match), + cost, totalCost); + totalCost += cost; #endif /* Encode Literal length */ - length = (int)(*ip - *anchor); - token = (*op)++; - if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ - if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } - else *token = (BYTE)(length<<ML_BITS); + length = (size_t)(*ip - *anchor); + if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ + if (length >= RUN_MASK) { + size_t len = length - RUN_MASK; + *token = (RUN_MASK << ML_BITS); + for(; len >= 255 ; len -= 255) *(*op)++ = 255; + *(*op)++ = (BYTE)len; + } else { + *token = (BYTE)(length << ML_BITS); + } /* Copy Literals */ LZ4_wildCopy(*op, *anchor, (*op) + length); @@ -286,13 +370,14 @@ FORCE_INLINE int LZ4HC_encodeSequence ( LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2; /* Encode MatchLength */ - length = (int)(matchLength-MINMATCH); - if ((limitedOutputBuffer) && (*op + (length>>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ - if (length>=(int)ML_MASK) { + assert(matchLength >= MINMATCH); + length = (size_t)(matchLength - MINMATCH); + if ((limit) && (*op + (length >> 8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ + if (length >= ML_MASK) { *token += ML_MASK; length -= ML_MASK; - for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } - if (length > 254) { length-=255; *(*op)++ = 255; } + for(; length >= 510 ; length -= 510) { *(*op)++ = 255; *(*op)++ = 255; } + if (length >= 255) { length -= 255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } else { *token += (BYTE)(length); @@ -305,27 +390,33 @@ FORCE_INLINE int LZ4HC_encodeSequence ( return 0; } +/* btopt */ +#include "lz4opt.h" -static int LZ4HC_compress_generic ( + +static int LZ4HC_compress_hashChain ( LZ4HC_CCtx_internal* const ctx, const char* const source, char* const dest, - int const inputSize, + int* srcSizePtr, int const maxOutputSize, - int compressionLevel, + unsigned maxNbAttempts, limitedOutput_directive limit ) { + const int inputSize = *srcSizePtr; + const int patternAnalysis = (maxNbAttempts > 64); /* levels 8+ */ + const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; const BYTE* const iend = ip + inputSize; const BYTE* const mflimit = iend - MFLIMIT; const BYTE* const matchlimit = (iend - LASTLITERALS); + BYTE* optr = (BYTE*) dest; BYTE* op = (BYTE*) dest; - BYTE* const oend = op + maxOutputSize; + BYTE* oend = op + maxOutputSize; - unsigned maxNbAttempts; int ml, ml2, ml3, ml0; const BYTE* ref = NULL; const BYTE* start2 = NULL; @@ -336,17 +427,14 @@ static int LZ4HC_compress_generic ( const BYTE* ref0; /* init */ - if (compressionLevel > LZ4HC_MAX_CLEVEL) compressionLevel = LZ4HC_MAX_CLEVEL; - if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; - maxNbAttempts = 1 << (compressionLevel-1); - ctx->end += inputSize; - - ip++; + *srcSizePtr = 0; + if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ /* Main Loop */ while (ip < mflimit) { - ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); - if (!ml) { ip++; continue; } + ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis); + if (ml<MINMATCH) { ip++; continue; } /* saved, in case we would skip too much */ start0 = ip; @@ -355,11 +443,15 @@ static int LZ4HC_compress_generic ( _Search2: if (ip+ml < mflimit) - ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts); - else ml2 = ml; + ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, + ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, + maxNbAttempts, patternAnalysis); + else + ml2 = ml; if (ml2 == ml) { /* No better match */ - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + optr = op; + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; continue; } @@ -380,11 +472,9 @@ _Search2: } _Search3: - /* - * Currently we have : - * ml2 > ml1, and - * ip1+3 <= ip2 (usually < ip1+ml1) - */ + /* At this stage, we have : + * ml2 > ml1, and + * ip1+3 <= ip2 (usually < ip1+ml1) */ if ((start2 - ip) < OPTIMAL_ML) { int correction; int new_ml = ml; @@ -400,16 +490,21 @@ _Search3: /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ if (start2 + ml2 < mflimit) - ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); - else ml3 = ml2; + ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, + start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, + maxNbAttempts, patternAnalysis); + else + ml3 = ml2; if (ml3 == ml2) { /* No better match : 2 sequences to encode */ /* ip & ref are known; Now for ml */ if (start2 < ip+ml) ml = (int)(start2 - ip); /* Now, encode 2 sequences */ - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + optr = op; + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; ip = start2; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0; + optr = op; + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) goto _dest_overflow; continue; } @@ -427,7 +522,8 @@ _Search3: } } - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + optr = op; + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; ip = start3; ref = ref3; ml = ml3; @@ -463,7 +559,8 @@ _Search3: ml = (int)(start2 - ip); } } - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + optr = op; + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; ip = start2; ref = ref2; @@ -476,34 +573,114 @@ _Search3: goto _Search3; } +_last_literals: /* Encode Last Literals */ - { int lastRun = (int)(iend - anchor); - if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (BYTE)(lastRun<<ML_BITS); - memcpy(op, anchor, iend - anchor); - op += iend-anchor; + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + litLength + lastRunSize; + if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) return 0; /* Check output limit */ + /* adapt lastRunSize to fill 'dest' */ + lastRunSize = (size_t)(oend - op) - 1; + litLength = (lastRunSize + 255 - RUN_MASK) / 255; + lastRunSize -= litLength; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + memcpy(op, anchor, lastRunSize); + op += lastRunSize; } /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); return (int) (((char*)op)-dest); + +_dest_overflow: + if (limit == limitedDestSize) { + op = optr; /* restore correct out pointer */ + goto _last_literals; + } + return 0; +} + + +static int LZ4HC_compress_generic ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + typedef enum { lz4hc, lz4opt } lz4hc_strat_e; + typedef struct { + lz4hc_strat_e strat; + U32 nbSearches; + U32 targetLength; + } cParams_t; + static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = { + { lz4hc, 2, 16 }, /* 0, unused */ + { lz4hc, 2, 16 }, /* 1, unused */ + { lz4hc, 2, 16 }, /* 2, unused */ + { lz4hc, 4, 16 }, /* 3 */ + { lz4hc, 8, 16 }, /* 4 */ + { lz4hc, 16, 16 }, /* 5 */ + { lz4hc, 32, 16 }, /* 6 */ + { lz4hc, 64, 16 }, /* 7 */ + { lz4hc, 128, 16 }, /* 8 */ + { lz4hc, 256, 16 }, /* 9 */ + { lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/ + { lz4opt, 512,128 }, /*11 */ + { lz4opt,8192, LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ + }; + + if (limit == limitedDestSize && dstCapacity < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ + + ctx->end += *srcSizePtr; + if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe something to review */ + cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel); + assert(cLevel >= 0); + assert(cLevel <= LZ4HC_CLEVEL_MAX); + { cParams_t const cParam = clTable[cLevel]; + if (cParam.strat == lz4hc) + return LZ4HC_compress_hashChain(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, limit); + assert(cParam.strat == lz4opt); + return LZ4HC_compress_optimal(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, cParam.targetLength, limit, + cLevel == LZ4HC_CLEVEL_MAX); /* ultra mode */ + } } int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } -int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) +int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { - LZ4HC_CCtx_internal* ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; + LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ LZ4HC_init (ctx, (const BYTE*)src); - if (maxDstSize < LZ4_compressBound(srcSize)) - return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput); + if (dstCapacity < LZ4_compressBound(srcSize)) + return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput); else - return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, noLimit); + return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit); } -int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) +int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); @@ -511,13 +688,22 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int LZ4_streamHC_t state; LZ4_streamHC_t* const statePtr = &state; #endif - int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); + int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 free(statePtr); #endif return cSize; } +/* LZ4_compress_HC_destSize() : + * only compatible with regular HC parser */ +int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) +{ + LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; + LZ4HC_init(ctx, (const BYTE*) source); + return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize); +} + /************************************** @@ -525,7 +711,11 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int **************************************/ /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } -int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; } +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { + if (!LZ4_streamHCPtr) return 0; /* support free on NULL */ + free(LZ4_streamHCPtr); + return 0; +} /* initialization */ @@ -533,19 +723,26 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ LZ4_streamHCPtr->internal_donotuse.base = NULL; - LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel; + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); +} + +void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + if (compressionLevel < 1) compressionLevel = 1; + if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; + LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel; } int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { - LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; } LZ4HC_init (ctxPtr, (const BYTE*)dictionary); - if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3)); ctxPtr->end = (const BYTE*)dictionary + dictSize; + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); return dictSize; } @@ -555,6 +752,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); @@ -565,12 +763,13 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl } static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, - const char* source, char* dest, - int inputSize, int maxOutputSize, limitedOutput_directive limit) + const char* src, char* dst, + int* srcSizePtr, int dstCapacity, + limitedOutput_directive limit) { - LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; /* auto-init if forgotten */ - if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source); + if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src); /* Check overflow */ if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { @@ -580,30 +779,36 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, } /* Check if blocks follow each other */ - if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source); + if ((const BYTE*)src != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src); /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; + { const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr; const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; - if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) { + if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) { if (sourceEnd > dictEnd) sourceEnd = dictEnd; ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; } } - return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit); + return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit); } -int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) +int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity) { - if (maxOutputSize < LZ4_compressBound(inputSize)) - return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); + if (dstCapacity < LZ4_compressBound(srcSize)) + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput); else - return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, noLimit); } +int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize) +{ + return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, limitedDestSize); +} + + /* dictionary saving */ @@ -664,16 +869,20 @@ void* LZ4_createHC (char* inputBuffer) return hc4; } -int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; } +int LZ4_freeHC (void* LZ4HC_Data) { + if (!LZ4HC_Data) return 0; /* support free on NULL */ + FREEMEM(LZ4HC_Data); + return 0; +} -int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) +int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel) { - return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, 0, compressionLevel, noLimit); + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, noLimit); } -int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) +int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel) { - return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, dstCapacity, cLevel, limitedOutput); } char* LZ4_slideInputBufferHC(void* LZ4HC_Data) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index d8ac3c0..d41bf42 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -1,7 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 Header File - Copyright (C) 2011-2016, Yann Collet. + Copyright (C) 2011-2017, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -39,26 +39,27 @@ extern "C" { #endif /* --- Dependency --- */ -/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ +/* note : lz4hc requires lz4.h/lz4.c for compilation */ #include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ /* --- Useful constants --- */ -#define LZ4HC_MIN_CLEVEL 3 -#define LZ4HC_DEFAULT_CLEVEL 9 -#define LZ4HC_MAX_CLEVEL 16 +#define LZ4HC_CLEVEL_MIN 3 +#define LZ4HC_CLEVEL_DEFAULT 9 +#define LZ4HC_CLEVEL_OPT_MIN 10 +#define LZ4HC_CLEVEL_MAX 12 /*-************************************ * Block Compression **************************************/ /*! LZ4_compress_HC() : - * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm. + * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm. * `dst` must be already allocated. - * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") - * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") - * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work. - * Values >LZ4HC_MAX_CLEVEL behave the same as 16. + * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") + * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") + * `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work. + * Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX. * @return : the number of bytes written into 'dst' * or 0 if compression fails. */ @@ -71,12 +72,12 @@ LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dst /*! LZ4_compress_HC_extStateHC() : - * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. + * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. * `state` size is provided by LZ4_sizeofStateHC(). - * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). + * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly). */ -LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); LZ4LIB_API int LZ4_sizeofStateHC(void); +LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); /*-************************************ @@ -86,10 +87,10 @@ LZ4LIB_API int LZ4_sizeofStateHC(void); typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */ /*! LZ4_createStreamHC() and LZ4_freeStreamHC() : - * These functions create and release memory for LZ4 HC streaming state. - * Newly created states are automatically initialized. - * Existing states can be re-used several times, using LZ4_resetStreamHC(). - * These methods are API and ABI stable, they can be used in combination with a DLL. + * These functions create and release memory for LZ4 HC streaming state. + * Newly created states are automatically initialized. + * Existing states can be re-used several times, using LZ4_resetStreamHC(). + * These methods are API and ABI stable, they can be used in combination with a DLL. */ LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); @@ -112,29 +113,27 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in Then, use LZ4_compress_HC_continue() to compress each successive block. Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. - 'dst' buffer should be sized to handle worst case scenarios, using LZ4_compressBound(), to ensure operation success. + 'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), to ensure operation success. + Because in case of failure, the API does not guarantee context recovery, and context will have to be reset. + If `dst` buffer budget cannot be >= LZ4_compressBound(), consider using LZ4_compress_HC_continue_destSize() instead. - If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block, - you must save it to a safer memory space, using LZ4_saveDictHC(). + If, for any reason, previous data block can't be preserved unmodified in memory for next compression block, + you can save it to a more stable memory space, using LZ4_saveDictHC(). Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'. */ -/*-****************************************** - * !!!!! STATIC LINKING ONLY !!!!! - *******************************************/ - - /*-************************************* +/*-************************************************************** * PRIVATE DEFINITIONS : * Do not use these definitions. * They are exposed to allow static allocation of `LZ4_streamHC_t`. * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 - **************************************/ + ****************************************************************/ #define LZ4HC_DICTIONARY_LOGSIZE 16 #define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE) #define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) -#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) +#define LZ4HC_HASH_LOG 15 #define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG) #define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) @@ -146,14 +145,14 @@ typedef struct { uint32_t hashTable[LZ4HC_HASHTABLESIZE]; uint16_t chainTable[LZ4HC_MAXD]; - const uint8_t* end; /* next block here to continue on current prefix */ - const uint8_t* base; /* All index relative to this position */ - const uint8_t* dictBase; /* alternate base for extDict */ - uint8_t* inputBuffer; /* deprecated */ - uint32_t dictLimit; /* below that point, need extDict */ - uint32_t lowLimit; /* below that point, no more dict */ - uint32_t nextToUpdate; /* index from which to continue dictionary update */ - uint32_t compressionLevel; + const uint8_t* end; /* next block here to continue on current prefix */ + const uint8_t* base; /* All index relative to this position */ + const uint8_t* dictBase; /* alternate base for extDict */ + uint8_t* inputBuffer; /* deprecated */ + uint32_t dictLimit; /* below that point, need extDict */ + uint32_t lowLimit; /* below that point, no more dict */ + uint32_t nextToUpdate; /* index from which to continue dictionary update */ + int compressionLevel; } LZ4HC_CCtx_internal; #else @@ -161,7 +160,7 @@ typedef struct typedef struct { unsigned int hashTable[LZ4HC_HASHTABLESIZE]; - unsigned short chainTable[LZ4HC_MAXD]; + unsigned short chainTable[LZ4HC_MAXD]; const unsigned char* end; /* next block here to continue on current prefix */ const unsigned char* base; /* All index relative to this position */ const unsigned char* dictBase; /* alternate base for extDict */ @@ -169,12 +168,12 @@ typedef struct unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ - unsigned int compressionLevel; + int compressionLevel; } LZ4HC_CCtx_internal; #endif -#define LZ4_STREAMHCSIZE 262192 +#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */ #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) union LZ4_streamHC_u { size_t table[LZ4_STREAMHCSIZE_SIZET]; @@ -196,26 +195,25 @@ union LZ4_streamHC_u { /* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */ /* deprecated compression functions */ -/* these functions will trigger warning messages in future releases */ -LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* Deprecated Streaming functions using older model; should no longer be used */ -LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer); -LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data); -LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void); -LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void); +LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer); #if defined (__cplusplus) @@ -223,3 +221,52 @@ LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC( #endif #endif /* LZ4_HC_H_19834876238432 */ + + +/*-************************************************** + * !!!!! STATIC LINKING ONLY !!!!! + * Following definitions are considered experimental. + * They should not be linked from DLL, + * as there is no guarantee of API stability yet. + * Prototypes will be promoted to "stable" status + * after successfull usage in real-life scenarios. + ***************************************************/ +#ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */ +#ifndef LZ4_HC_SLO_098092834 +#define LZ4_HC_SLO_098092834 + +/*! LZ4_compress_HC_destSize() : v1.8.0 (experimental) + * Will try to compress as much data from `src` as possible + * that can fit into `targetDstSize` budget. + * Result is provided in 2 parts : + * @return : the number of bytes written into 'dst' + * or 0 if compression fails. + * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src` + */ +int LZ4_compress_HC_destSize(void* LZ4HC_Data, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize, + int compressionLevel); + +/*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental) + * Similar as LZ4_compress_HC_continue(), + * but will read a variable nb of bytes from `src` + * to fit into `targetDstSize` budget. + * Result is provided in 2 parts : + * @return : the number of bytes written into 'dst' + * or 0 if compression fails. + * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`. + */ +int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize); + +/*! LZ4_setCompressionLevel() : v1.8.0 (experimental) + * It's possible to change compression level between 2 invocations of LZ4_compress_HC_continue*() + */ +void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); + + + +#endif /* LZ4_HC_SLO_098092834 */ +#endif /* LZ4_HC_STATIC_LINKING_ONLY */ diff --git a/lib/lz4opt.h b/lib/lz4opt.h new file mode 100644 index 0000000..5a8438c --- /dev/null +++ b/lib/lz4opt.h @@ -0,0 +1,356 @@ +/* + lz4opt.h - Optimal Mode of LZ4 + Copyright (C) 2015-2017, Przemyslaw Skibinski <inikep@gmail.com> + Note : this file is intended to be included within lz4hc.c + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +#define LZ4_OPT_NUM (1<<12) + +typedef struct { + int price; + int off; + int mlen; + int litlen; +} LZ4HC_optimal_t; + + +/* price in bytes */ +LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) +{ + int price = litlen; + if (litlen >= (int)RUN_MASK) + price += 1 + (litlen-RUN_MASK)/255; + return price; +} + + +/* requires mlen >= MINMATCH */ +LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) +{ + int price = 1 + 2 ; /* token + 16-bit offset */ + + price += LZ4HC_literalsPrice(litlen); + + if (mlen >= (int)(ML_MASK+MINMATCH)) + price += 1 + (mlen-(ML_MASK+MINMATCH))/255; + + return price; +} + + +/*-************************************* +* Match finder +***************************************/ +typedef struct { + int off; + int len; +} LZ4HC_match_t; + +LZ4_FORCE_INLINE +LZ4HC_match_t LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, + const BYTE* ip, const BYTE* const iHighLimit, + int minLen, int nbSearches) +{ + LZ4HC_match_t match = { 0 , 0 }; + const BYTE* matchPtr = NULL; + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, + ip, ip, iHighLimit, minLen, &matchPtr, &ip, + nbSearches, 1 /* patternAnalysis */); + if (matchLength <= minLen) return match; + match.len = matchLength; + match.off = (int)(ip-matchPtr); + return match; +} + + +static int LZ4HC_compress_optimal ( + LZ4HC_CCtx_internal* ctx, + const char* const source, + char* dst, + int* srcSizePtr, + int dstCapacity, + int const nbSearches, + size_t sufficient_len, + limitedOutput_directive limit, + int const fullUpdate + ) +{ +#define TRAILING_LITERALS 3 + LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* this uses a bit too much stack memory to my taste ... */ + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + BYTE* op = (BYTE*) dst; + BYTE* opSaved = (BYTE*) dst; + BYTE* oend = op + dstCapacity; + + /* init */ + DEBUGLOG(5, "LZ4HC_compress_optimal"); + *srcSizePtr = 0; + if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; + + /* Main Loop */ + assert(ip - anchor < LZ4_MAX_INPUT_SIZE); + while (ip < mflimit) { + int const llen = (int)(ip - anchor); + int best_mlen, best_off; + int cur, last_match_pos = 0; + + LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches); + if (firstMatch.len==0) { ip++; continue; } + + if ((size_t)firstMatch.len > sufficient_len) { + /* good enough solution : immediate encoding */ + int const firstML = firstMatch.len; + const BYTE* const matchPos = ip - firstMatch.off; + opSaved = op; + if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ + goto _dest_overflow; + continue; + } + + /* set prices for first positions (literals) */ + { int rPos; + for (rPos = 0 ; rPos < MINMATCH ; rPos++) { + int const cost = LZ4HC_literalsPrice(llen + rPos); + opt[rPos].mlen = 1; + opt[rPos].off = 0; + opt[rPos].litlen = llen + rPos; + opt[rPos].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + rPos, cost, opt[rPos].litlen); + } } + /* set prices using initial match */ + { int mlen = MINMATCH; + int const matchML = firstMatch.len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ + int const offset = firstMatch.off; + assert(matchML < LZ4_OPT_NUM); + for ( ; mlen <= matchML ; mlen++) { + int const cost = LZ4HC_sequencePrice(llen, mlen); + opt[mlen].mlen = mlen; + opt[mlen].off = offset; + opt[mlen].litlen = llen; + opt[mlen].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup", + mlen, cost, mlen); + } } + last_match_pos = firstMatch.len; + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + + /* check further positions */ + for (cur = 1; cur < last_match_pos; cur++) { + const BYTE* const curPtr = ip + cur; + LZ4HC_match_t newMatch; + + if (curPtr >= mflimit) break; + DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u", + cur, opt[cur].price, opt[cur+1].price, cur+1); + if (fullUpdate) { + /* not useful to search here if next position has same (or lower) cost */ + if ( (opt[cur+1].price <= opt[cur].price) + /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */ + && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) ) + continue; + } else { + /* not useful to search here if next position has same (or lower) cost */ + if (opt[cur+1].price <= opt[cur].price) continue; + } + + DEBUGLOG(7, "search at rPos:%u", cur); + if (fullUpdate) + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches); + else + /* only test matches of minimum length; slightly faster, but misses a few bytes */ + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches); + if (!newMatch.len) continue; + + if ( ((size_t)newMatch.len > sufficient_len) + || (newMatch.len + cur >= LZ4_OPT_NUM) ) { + /* immediate encoding */ + best_mlen = newMatch.len; + best_off = newMatch.off; + last_match_pos = cur + 1; + goto encode; + } + + /* before match : set price with literals at beginning */ + { int const baseLitlen = opt[cur].litlen; + int litlen; + for (litlen = 1; litlen < MINMATCH; litlen++) { + int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen); + int const pos = cur + litlen; + if (price < opt[pos].price) { + opt[pos].mlen = 1; /* literal */ + opt[pos].off = 0; + opt[pos].litlen = baseLitlen+litlen; + opt[pos].price = price; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", + pos, price, opt[pos].litlen); + } } } + + /* set prices using match at position = cur */ + { int const matchML = newMatch.len; + int ml = MINMATCH; + + assert(cur + newMatch.len < LZ4_OPT_NUM); + for ( ; ml <= matchML ; ml++) { + int const pos = cur + ml; + int const offset = newMatch.off; + int price; + int ll; + DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)", + pos, last_match_pos); + if (opt[cur].mlen == 1) { + ll = opt[cur].litlen; + price = ((cur > ll) ? opt[cur - ll].price : 0) + + LZ4HC_sequencePrice(ll, ml); + } else { + ll = 0; + price = opt[cur].price + LZ4HC_sequencePrice(0, ml); + } + + if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) { + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)", + pos, price, ml); + assert(pos < LZ4_OPT_NUM); + if ( (ml == matchML) /* last pos of last match */ + && (last_match_pos < pos) ) + last_match_pos = pos; + opt[pos].mlen = ml; + opt[pos].off = offset; + opt[pos].litlen = ll; + opt[pos].price = price; + } } } + /* complete following positions with literals */ + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + } /* for (cur = 1; cur <= last_match_pos; cur++) */ + + best_mlen = opt[last_match_pos].mlen; + best_off = opt[last_match_pos].off; + cur = last_match_pos - best_mlen; + +encode: /* cur, last_match_pos, best_mlen, best_off must be set */ + assert(cur < LZ4_OPT_NUM); + assert(last_match_pos >= 1); /* == 1 when only one candidate */ + DEBUGLOG(6, "reverse traversal, looking for shortest path") + DEBUGLOG(6, "last_match_pos = %i", last_match_pos); + { int candidate_pos = cur; + int selected_matchLength = best_mlen; + int selected_offset = best_off; + while (1) { /* from end to beginning */ + int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */ + int const next_offset = opt[candidate_pos].off; + DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength); + opt[candidate_pos].mlen = selected_matchLength; + opt[candidate_pos].off = selected_offset; + selected_matchLength = next_matchLength; + selected_offset = next_offset; + if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */ + assert(next_matchLength > 0); /* can be 1, means literal */ + candidate_pos -= next_matchLength; + } } + + /* encode all recorded sequences in order */ + { int rPos = 0; /* relative position (to ip) */ + while (rPos < last_match_pos) { + int const ml = opt[rPos].mlen; + int const offset = opt[rPos].off; + if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */ + rPos += ml; + assert(ml >= MINMATCH); + assert((offset >= 1) && (offset <= MAX_DISTANCE)); + opSaved = op; + if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ + goto _dest_overflow; + } } + } /* while (ip < mflimit) */ + +_last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + litLength + lastRunSize; + if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) return 0; /* Check output limit */ + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (size_t)(oend - op) - 1; + litLength = (lastRunSize + 255 - RUN_MASK) / 255; + lastRunSize -= litLength; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); + return (int) ((char*)op-dst); + +_dest_overflow: + if (limit == limitedDestSize) { + op = opSaved; /* restore correct out pointer */ + goto _last_literals; + } + return 0; +} diff --git a/lib/xxhash.c b/lib/xxhash.c index e9ff2d4..bcf1f1d 100644 --- a/lib/xxhash.c +++ b/lib/xxhash.c @@ -113,19 +113,24 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp ***************************************/ #ifdef _MSC_VER /* Visual Studio */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# define FORCE_INLINE static __forceinline -#else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ #endif +#ifndef XXH_FORCE_INLINE +# ifdef _MSC_VER /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define XXH_FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define XXH_FORCE_INLINE static inline +# endif +# else +# define XXH_FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +# endif /* _MSC_VER */ +#endif /* XXH_FORCE_INLINE */ + /* ************************************* * Basic Types @@ -218,7 +223,7 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; *****************************/ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; -FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +XXH_FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); @@ -226,7 +231,7 @@ FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_a return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); } -FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +XXH_FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } @@ -261,7 +266,7 @@ static U32 XXH32_round(U32 seed, U32 input) return seed; } -FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; const BYTE* bEnd = p + len; @@ -376,7 +381,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int s } -FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +XXH_FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) { const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; @@ -446,7 +451,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* -FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +XXH_FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) { const BYTE * p = (const BYTE*)state->mem32; const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; @@ -578,7 +583,7 @@ static U64 XXH_swap64 (U64 x) } #endif -FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +XXH_FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); @@ -586,7 +591,7 @@ FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_a return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); } -FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +XXH_FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) { return XXH_readLE64_align(ptr, endian, XXH_unaligned); } @@ -621,7 +626,7 @@ static U64 XXH64_mergeRound(U64 acc, U64 val) return acc; } -FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +XXH_FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; @@ -745,7 +750,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long return XXH_OK; } -FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +XXH_FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) { const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; @@ -810,7 +815,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* return XXH64_update_endian(state_in, input, len, XXH_bigEndian); } -FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) +XXH_FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) { const BYTE * p = (const BYTE*)state->mem64; const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; diff --git a/programs/.gitignore b/programs/.gitignore index df3423e..daa7f14 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -1,5 +1,7 @@ # local binary (Makefile) lz4 +unlz4 +lz4cat lz4c lz4c32 datagen diff --git a/programs/Makefile b/programs/Makefile index 9939a44..a51bd4b 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -1,6 +1,8 @@ # ########################################################################## # LZ4 programs - Makefile -# Copyright (C) Yann Collet 2011-2016 +# Copyright (C) Yann Collet 2011-2017 +# +# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets # # GPL v2 License # @@ -27,27 +29,41 @@ # lz4c32: Same as lz4c, but forced to compile in 32-bits mode # ########################################################################## -DESTDIR ?= -PREFIX ?= /usr/local -BINDIR := $(PREFIX)/bin -MANDIR := $(PREFIX)/share/man/man1 -LZ4DIR := ../lib -VOID := /dev/null - -CFLAGS ?= -O3 # allows custom optimization flags. For example : CFLAGS="-O2 -g" make -CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ - -Wpointer-arith -Wstrict-aliasing=1 -CFLAGS += $(MOREFLAGS) -CPPFLAGS:= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ -FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) +# Version numbers +LZ4DIR := ../lib +LIBVER_SRC := $(LZ4DIR)/lz4.h +LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) +LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) +LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) + +SRCFILES := $(sort $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c)) +OBJFILES := $(SRCFILES:.c=.o) + +CPPFLAGS += -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ +CFLAGS ?= -O3 +DEBUGFLAGS:=-Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ + -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ + -Wpointer-arith -Wstrict-aliasing=1 +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) + +LZ4_VERSION=$(LIBVER) +MD2ROFF = ronn +MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)" # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) -EXT =.exe +EXT :=.exe +VOID := nul else -EXT = +EXT := +VOID := /dev/null endif @@ -59,50 +75,96 @@ all: lz4 lz4c all32: CFLAGS+=-m32 all32: all -lz4: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o +lz4: $(OBJFILES) $(CC) $(FLAGS) $^ -o $@$(EXT) -lz4-release: CFLAGS := -O3 +lz4-release: DEBUGFLAGS= lz4-release: lz4 -lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.c datagen.o - $(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) +lz4c: lz4 + ln -s lz4 lz4c + +lz4c32: CFLAGS += -m32 +lz4c32 : $(SRCFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) + +lz4.1: lz4.1.md $(LIBVER_SRC) + cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ + +man: lz4.1 + +clean-man: + rm lz4.1 -lz4c32: CFLAGS+=-m32 -lz4c32: lz4 - @cp lz4$(EXT) lz4c32$(EXT) +preview-man: clean-man man + man ./lz4.1 clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(RM) core *.o *.test tmp* \ - lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) + lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat @echo Cleaning completed -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#----------------------------------------------------------------------------- +# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) + +unlz4: lz4 + ln -s lz4 unlz4 + +lz4cat: lz4 + ln -s lz4 lz4cat + +DESTDIR ?= +# directory variables : GNU conventions prefer lowercase +# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html +# support both lower and uppercase (BSD), use uppercase in script +prefix ?= /usr/local +PREFIX ?= $(prefix) +exec_prefix ?= $(PREFIX) +bindir ?= $(exec_prefix)/bin +BINDIR ?= $(bindir) +datarootdir ?= $(PREFIX)/share +mandir ?= $(datarootdir)/man +man1dir ?= $(mandir)/man1 + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) +MANDIR ?= $(PREFIX)/man/man1 +else +MANDIR ?= $(man1dir) +endif + +ifneq (,$(filter $(shell uname),SunOS)) +INSTALL ?= ginstall +else +INSTALL ?= install +endif + +INSTALL_PROGRAM ?= $(INSTALL) -m 755 +INSTALL_DATA ?= $(INSTALL) -m 644 + -install: lz4$(EXT) lz4c$(EXT) +install: lz4 @echo Installing binaries - @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4$(EXT) - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4cat$(EXT) - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/unlz4$(EXT) - @install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c$(EXT) + @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ + @$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4 + @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4c + @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat + @ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4 @echo Installing man pages - @install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 + @$(INSTALL_DATA) lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1 @echo lz4 installation completed uninstall: - @$(RM) $(DESTDIR)$(BINDIR)/lz4cat$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/unlz4$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/lz4$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/lz4c$(EXT) + @$(RM) $(DESTDIR)$(BINDIR)/lz4cat + @$(RM) $(DESTDIR)$(BINDIR)/unlz4 + @$(RM) $(DESTDIR)$(BINDIR)/lz4 + @$(RM) $(DESTDIR)$(BINDIR)/lz4c @$(RM) $(DESTDIR)$(MANDIR)/lz4.1 @$(RM) $(DESTDIR)$(MANDIR)/lz4c.1 @$(RM) $(DESTDIR)$(MANDIR)/lz4cat.1 diff --git a/programs/bench.c b/programs/bench.c index 434da8b..fac2a87 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -24,13 +24,22 @@ */ +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + /* ************************************* * Includes ***************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ #include <stdlib.h> /* malloc, free */ #include <string.h> /* memset */ -#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <stdio.h> /* fprintf, fopen, ftello */ #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ #include "datagen.h" /* RDG_genBuffer */ @@ -59,7 +68,7 @@ static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSi #define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ #define COOLPERIOD_SEC 10 -#define DECOMP_MULT 2 /* test decompression DECOMP_MULT times longer than compression */ +#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */ #define KB *(1 <<10) #define MB *(1 <<20) @@ -157,7 +166,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, void* const compressedBuffer = malloc(maxCompressedSize); void* const resultBuffer = malloc(srcSize); U32 nbBlocks; - UTIL_time_t ticksPerSecond; struct compressionParameters compP; int cfunctionId; @@ -167,10 +175,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* init */ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ - UTIL_initTimer(&ticksPerSecond); /* Init */ - if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1; + if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1; switch (cfunctionId) { #ifdef COMPRESSOR0 @@ -220,17 +227,17 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, size_t cSize = 0; double ratio = 0.; - UTIL_getTime(&coolTime); + coolTime = UTIL_getTime(); DISPLAYLEVEL(2, "\r%79s\r", ""); - while (!cCompleted | !dCompleted) { + while (!cCompleted || !dCompleted) { UTIL_time_t clockStart; U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; /* overheat protection */ - if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) { + if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { DISPLAYLEVEL(2, "\rcooling down ... \r"); UTIL_sleep(COOLPERIOD_SEC); - UTIL_getTime(&coolTime); + coolTime = UTIL_getTime(); } /* Compression */ @@ -238,8 +245,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ UTIL_sleepMilli(1); /* give processor time to other processes */ - UTIL_waitForNextTick(ticksPerSecond); - UTIL_getTime(&clockStart); + UTIL_waitForNextTick(); + clockStart = UTIL_getTime(); if (!cCompleted) { /* still some time to do compression tests */ U32 nbLoops = 0; @@ -251,8 +258,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].cSize = rSize; } nbLoops++; - } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop); - { U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond); + } while (UTIL_clockSpanMicro(clockStart) < clockLoop); + { U64 const clockSpan = UTIL_clockSpanMicro(clockStart); if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops; totalCTime += clockSpan; cCompleted = totalCTime>maxTime; @@ -273,8 +280,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ UTIL_sleepMilli(1); /* give processor time to other processes */ - UTIL_waitForNextTick(ticksPerSecond); - UTIL_getTime(&clockStart); + UTIL_waitForNextTick(); + clockStart = UTIL_getTime(); if (!dCompleted) { U32 nbLoops = 0; @@ -291,8 +298,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].resSize = regenSize; } nbLoops++; - } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < DECOMP_MULT*clockLoop); - { U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond); + } while (UTIL_clockSpanMicro(clockStart) < DECOMP_MULT*clockLoop); + { U64 const clockSpan = UTIL_clockSpanMicro(clockStart); if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops; totalDTime += clockSpan; dCompleted = totalDTime>(DECOMP_MULT*maxTime); @@ -384,7 +391,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - SET_HIGH_PRIORITY; + SET_REALTIME_PRIORITY; if (g_displayLevel == 1 && !g_additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", LZ4_VERSION_STRING, LZ4_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); @@ -419,7 +426,10 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, f = fopen(fileNamesTable[n], "rb"); if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]); DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); - if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + if (fileSize > bufferSize-pos) { /* buffer too small - stop after this file */ + fileSize = bufferSize-pos; + nbFiles=n; + } { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); pos += readSize; } @@ -446,8 +456,13 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; if (benchedSize==0) EXM_THROW(12, "not enough memory"); if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; - if (benchedSize < totalSizeToLoad) - DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); + if (benchedSize > LZ4_MAX_INPUT_SIZE) { + benchedSize = LZ4_MAX_INPUT_SIZE; + DISPLAY("File(s) bigger than LZ4's max input size; testing %u MB only...\n", (U32)(benchedSize >> 20)); + } else { + if (benchedSize < totalSizeToLoad) + DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); + } srcBuffer = malloc(benchedSize + !benchedSize); /* avoid alloc of zero */ if (!srcBuffer) EXM_THROW(12, "not enough memory"); @@ -494,8 +509,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, { double const compressibility = (double)g_compressibilityDefault / 100; - if (cLevel > LZ4HC_MAX_CLEVEL) cLevel = LZ4HC_MAX_CLEVEL; - if (cLevelLast > LZ4HC_MAX_CLEVEL) cLevelLast = LZ4HC_MAX_CLEVEL; + if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; + if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); diff --git a/programs/datagen.c b/programs/datagen.c index f97c2d0..7285d69 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -26,43 +26,14 @@ /************************************** * Includes **************************************/ +#include "platform.h" /* Compiler options, SET_BINARY_MODE */ +#include "util.h" /* U32 */ #include <stdlib.h> /* malloc */ #include <stdio.h> /* FILE, fwrite */ #include <string.h> /* memcpy */ /************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** -* OS-specific Includes -**************************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include <fcntl.h> /* _O_BINARY */ -# include <io.h> /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) -#else -# define SET_BINARY_MODE(file) -#endif - - -/************************************** * Constants **************************************/ #define KB *(1 <<10) @@ -148,7 +119,10 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match } /* init */ - if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1; + if (pos==0) { + buffPtr[0] = RDG_genChar(seed, lt); + pos=1; + } /* Generate compressible data */ while (pos < buffSize) diff --git a/programs/lz4.1 b/programs/lz4.1 index 2c94033..e0f6a81 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -1,251 +1,223 @@ -\." -\." lz4.1: This is a manual page for 'lz4' program. This file is part of the -\." lz4 <http://www.lz4.org/> project. -\." Author: Yann Collet -\." -. -\." No hyphenation -.hy 0 -.nr HY 0 -. -.TH lz4 "1" "2015-03-21" "lz4" "User Commands" -.SH NAME -\fBlz4, unlz4, lz4cat\fR \- Compress or decompress .lz4 files - -.SH SYNOPSIS -.TP 5 -\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE> -.PP -.B unlz4 -is equivalent to -.BR "lz4 \-d" -.br -.B lz4cat -is equivalent to -.BR "lz4 \-dcfm" -.br -.PP -When writing scripts that need to decompress files, -it is recommended to always use the name -.B lz4 -with appropriate arguments -.RB ( "lz4 \-d" -or -.BR "lz4 \-dc" ) -instead of the names -.B unlz4 -and -.BR lz4cat . - - -.SH DESCRIPTION -.PP -\fBlz4\fR is an extremely fast lossless compression algorithm, -based on \fBbyte-aligned LZ77\fR family of compression scheme. -\fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs. -It features an extremely fast decoder, with speed in multiple GB/s per core, -typically reaching RAM speed limit on multi-core systems. -The native file format is the -.B .lz4 -format. - -.B lz4 -supports a command line syntax similar \fIbut not identical\fR to -.BR gzip (1). -Differences are : - \fBlz4\fR preserves original files - \fBlz4\fR compresses a single file by default (use \fB-m\fR for multiple files) - \fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 - When no destination name is provided, compressed file name receives a \fB.lz4\fR suffix - When no destination name is provided, if \fBstdout\fR is \fInot\fR the console, it becomes the output (like a silent \fB-c\fR) - Therefore \fBlz4 file > /dev/null\fR will not create \fBfile.lz4\fR - \fBlz4 file\fR shows real-time statistics during compression (use \fB-q\fR to silent them) - -Default behaviors can be modified by opt-in commands, described below. -\fBlz4 --quiet --multiple\fR more closely mimics \fBgzip\fR behavior. - -.SS "Concatenation of .lz4 files" -It is possible to concatenate -.B .lz4 -files as is. -.B lz4 -will decompress such files as if they were a single -.B .lz4 -file. For example: - lz4 file1 > foo.lz4 - lz4 file2 >> foo.lz4 - then - lz4cat foo.lz4 - is equivalent to : - cat file1 file2 - -.PP +. +.TH "LZ4" "1" "2018-01-13" "lz4 1.8.1" "User Commands" +. +.SH "NAME" +\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files +. +.SH "SYNOPSIS" +\fBlz4\fR [\fIOPTIONS\fR] [\-|INPUT\-FILE] \fIOUTPUT\-FILE\fR +. +.P +\fBunlz4\fR is equivalent to \fBlz4 \-d\fR +. +.P +\fBlz4cat\fR is equivalent to \fBlz4 \-dcfm\fR +. +.P +When writing scripts that need to decompress files, it is recommended to always use the name \fBlz4\fR with appropriate arguments (\fBlz4 \-d\fR or \fBlz4 \-dc\fR) instead of the names \fBunlz4\fR and \fBlz4cat\fR\. +. +.SH "DESCRIPTION" +\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\. +. +.SS "Difference between lz4 and gzip" +\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are : +. +.IP "\(bu" 4 +\fBlz4\fR preserves original files +. +.IP "\(bu" 4 +\fBlz4\fR compresses a single file by default (see \fB\-m\fR for multiple files) +. +.IP "\(bu" 4 +\fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 +. +.IP "\(bu" 4 +\fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression) +. +.IP "\(bu" 4 +\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them) +. +.IP "\(bu" 4 +If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\. +. +.IP "\(bu" 4 +If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\. +. +.IP "\(bu" 4 +As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create \fBfile\.lz4\fR\. +. +.IP "\(bu" 4 +Another consequence of those rules is that to run \fBlz4\fR under \fBnohup\fR, you should provide a destination file: \fBnohup lz4 file file\.lz4\fR, because \fBnohup\fR writes the specified command\'s output to a file\. +. +.IP "" 0 +. +.P +Default behaviors can be modified by opt\-in commands, detailed below\. +. +.IP "\(bu" 4 +\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\. +. +.IP "\(bu" 4 +Similarly, \fBlz4 \-m \-d\fR can decompress multiple \fB*\.lz4\fR files\. +. +.IP "\(bu" 4 +It\'s possible to opt\-in to erase source files on successful compression or decompression, using \fB\-\-rm\fR command\. +. +.IP "\(bu" 4 +Consequently, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\. +. +.IP "" 0 +. +.SS "Concatenation of \.lz4 files" +It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example: +. +.IP "" 4 +. +.nf -.SH OPTIONS +lz4 file1 > foo\.lz4 +lz4 file2 >> foo\.lz4 +. +.fi +. +.IP "" 0 +. +.P +Then \fBlz4cat foo\.lz4\fR is equivalent to \fBcat file1 file2\fR\. +. +.SH "OPTIONS" . .SS "Short commands concatenation" -In some cases, some options can be expressed using short command -.B "-x" -or long command -.B "--long-word" . -Short commands can be concatenated together. For example, -.B "-d -c" -is equivalent to -.B "-dc" . -Long commands cannot be concatenated. -They must be clearly separated by a space. +In some cases, some options can be expressed using short command \fB\-x\fR or long command \fB\-\-long\-word\fR\. Short commands can be concatenated together\. For example, \fB\-d \-c\fR is equivalent to \fB\-dc\fR\. Long commands cannot be concatenated\. They must be clearly separated by a space\. +. .SS "Multiple commands" -When multiple contradictory commands are issued on a same command line, -only the latest one will be applied. +When multiple contradictory commands are issued on a same command line, only the latest one will be applied\. . .SS "Operation mode" +. .TP -.BR \-z ", " \-\-compress -Compress. -This is the default operation mode -when no operation mode option is specified , -no other operation mode is implied from the command name -(for example, -.B unlz4 -implies -.B \-\-decompress ), -nor from the input file name -(for example, a file extension -.B .lz4 -implies -.B \-\-decompress -by default). -.B -z -can also be used to force compression of an already compressed -.B .lz4 -file. -.TP -.BR \-d ", " \-\-decompress ", " \-\-uncompress -Decompress. -.B --decompress -is also the default operation when the input filename has an -.B .lz4 -extension. -.TP -.BR \-t ", " \-\-test -Test the integrity of compressed -.B .lz4 -files. -The decompressed data is discarded. -No files are created nor removed. - +\fB\-z\fR \fB\-\-compress\fR +Compress\. This is the default operation mode when no operation mode option is specified, no other operation mode is implied from the command name (for example, \fBunlz4\fR implies \fB\-\-decompress\fR), nor from the input file name (for example, a file extension \fB\.lz4\fR implies \fB\-\-decompress\fR by default)\. \fB\-z\fR can also be used to force compression of an already compressed \fB\.lz4\fR file\. +. .TP -.BR \-b# -Benchmark mode, using # compression level. - +\fB\-d\fR \fB\-\-decompress\fR \fB\-\-uncompress\fR +Decompress\. \fB\-\-decompress\fR is also the default operation when the input filename has an \fB\.lz4\fR extension\. +. +.TP +\fB\-t\fR \fB\-\-test\fR +Test the integrity of compressed \fB\.lz4\fR files\. The decompressed data is discarded\. No files are created nor removed\. +. +.TP +\fB\-b#\fR +Benchmark mode, using \fB#\fR compression level\. . .SS "Operation modifiers" +. .TP -.B \-# - compression level, with # being any value from 1 to 16. - Higher values trade compression speed for compression ratio. - Values above 16 are considered the same as 16. - Recommended values are 1 for fast compression (default), and 9 for high compression. - Speed/compression trade-off will vary depending on data to compress. - Decompression speed remains fast at all settings. - +\fB\-#\fR +Compression level, with # being any value from 1 to 16\. Higher values trade compression speed for compression ratio\. Values above 16 are considered the same as 16\. Recommended values are 1 for fast compression (default), and 9 for high compression\. Speed/compression trade\-off will vary depending on data to compress\. Decompression speed remains fast at all settings\. +. .TP -.BR \-f ", " --[no-]force - This option has several effects: -.RS -.IP \(bu 3 -If the target file already exists, -overwrite it without prompting. -.IP \(bu 3 -When used with -.B \-\-decompress -and -.B lz4 -cannot recognize the type of the source file, -copy the source file as is to standard output. -This allows -.B lz4cat -.B \-\-force -to be used like -.BR cat (1) -for files that have not been compressed with -.BR lz4 . -.RE - +\fB\-f\fR \fB\-\-[no\-]force\fR +This option has several effects: +. +.IP +If the target file already exists, overwrite it without prompting\. +. +.IP +When used with \fB\-\-decompress\fR and \fBlz4\fR cannot recognize the type of the source file, copy the source file as is to standard output\. This allows \fBlz4cat \-\-force\fR to be used like \fBcat (1)\fR for files that have not been compressed with \fBlz4\fR\. +. .TP -.BR \-c ", " \--stdout ", " \--to-stdout - force write to standard output, even if it is the console - +\fB\-c\fR \fB\-\-stdout\fR \fB\-\-to\-stdout\fR +Force write to standard output, even if it is the console\. +. .TP -.BR \-m ", " \--multiple - Multiple file names. - By default, the second filename is used as the destination filename for the compressed file. - With -.B -m -, it is possible to specify any number of input filenames. Each of them will be compressed -independently, and the resulting name of each compressed file will be -.B filename.lz4 +\fB\-m\fR \fB\-\-multiple\fR +Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\. . - .TP -.B \-B# - block size [4-7](default : 7) - B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB +\fB\-r\fR +operate recursively on directories\. This mode also sets \fB\-m\fR (multiple input files)\. +. .TP -.B \-BD - block dependency (improves compression ratio on small blocks) +\fB\-B#\fR +Block size [4\-7](default : 7) +. +.br +\fB\-B4\fR= 64KB ; \fB\-B5\fR= 256KB ; \fB\-B6\fR= 1MB ; \fB\-B7\fR= 4MB +. .TP -.B \--[no-]frame-crc - select frame checksum (default:enabled) +\fB\-BD\fR +Block Dependency (improves compression ratio on small blocks) +. .TP -.B \--[no-]content-size - header includes original size (default:not present) - Note : this option can only be activated when the original size can be determined, -hence for a file. It won't work with unknown source size, such as stdin or pipe. +\fB\-\-[no\-]frame\-crc\fR +Select frame checksum (default:enabled) +. .TP -.B \--[no-]sparse - sparse mode support (default:enabled on file, disabled on stdout) +\fB\-\-[no\-]content\-size\fR +Header includes original size (default:not present) +. +.br +Note : this option can only be activated when the original size can be determined, hence for a file\. It won\'t work with unknown source size, such as stdin or pipe\. +. .TP -.B \-l - use Legacy format (typically used for Linux Kernel compression) - note : \fB-l\fR is not compatible with \fB-m\fR (\fB--multiple\fR) +\fB\-\-[no\-]sparse\fR +Sparse mode support (default:enabled on file, disabled on stdout) +. +.TP +\fB\-l\fR +Use Legacy format (typically for Linux Kernel compression) +. +.br +Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) nor \fB\-r\fR . .SS "Other options" +. .TP -.BR \-v ", " --verbose - verbose mode +\fB\-v\fR \fB\-\-verbose\fR +Verbose mode +. .TP -.BR \-q ", " --quiet - suppress warnings and real-time statistics; specify twice to suppress errors too +\fB\-q\fR \fB\-\-quiet\fR +Suppress warnings and real\-time statistics; specify twice to suppress errors too +. .TP -.B \-h/\-H ", " --help - display help/long help and exit +\fB\-h\fR \fB\-H\fR \fB\-\-help\fR +Display help/long help and exit +. .TP -.BR \-V ", " \--version - display Version number and exit +\fB\-V\fR \fB\-\-version\fR +Display Version number and exit +. .TP -.BR \-k ", " \--keep - Don't delete source file. -This is default behavior anyway, so this option is just for compatibility with gzip/xz. - +\fB\-k\fR \fB\-\-keep\fR +Preserve source files (default behavior) +. +.TP +\fB\-\-rm\fR +Delete source files on successful compression or decompression . -.SS "Benchmark mode" .TP -.B \-b# - benchmark file(s), using # compression level +\fB\-\-\fR +Treat all subsequent arguments as files +. +.SS "Benchmark mode" +. .TP -.B \-e# - benchmark multiple compression levels, from b# to e# (included) +\fB\-b#\fR +Benchmark file(s), using # compression level +. .TP -.B \-i# - minimum evaluation in seconds [1-9] (default : 3) +\fB\-e#\fR +Benchmark multiple compression levels, from b# to e# (included) +. .TP -.B \-r - operate recursively on directories - - -.SH BUGS -Report bugs at: https://github.com/Cyan4973/lz4/issues - -.SH AUTHOR +\fB\-i#\fR +Minimum evaluation in seconds [1\-9] (default : 3) +. +.SH "BUGS" +Report bugs at: https://github\.com/lz4/lz4/issues +. +.SH "AUTHOR" Yann Collet diff --git a/programs/lz4.1.md b/programs/lz4.1.md new file mode 100644 index 0000000..b7b8570 --- /dev/null +++ b/programs/lz4.1.md @@ -0,0 +1,221 @@ +lz4(1) -- lz4, unlz4, lz4cat - Compress or decompress .lz4 files +================================================================ + +SYNOPSIS +-------- + +`lz4` [*OPTIONS*] [-|INPUT-FILE] <OUTPUT-FILE> + +`unlz4` is equivalent to `lz4 -d` + +`lz4cat` is equivalent to `lz4 -dcfm` + +When writing scripts that need to decompress files, +it is recommended to always use the name `lz4` with appropriate arguments +(`lz4 -d` or `lz4 -dc`) instead of the names `unlz4` and `lz4cat`. + + +DESCRIPTION +----------- + +`lz4` is an extremely fast lossless compression algorithm, +based on **byte-aligned LZ77** family of compression scheme. +`lz4` offers compression speeds of 400 MB/s per core, linearly scalable with +multi-core CPUs. +It features an extremely fast decoder, with speed in multiple GB/s per core, +typically reaching RAM speed limit on multi-core systems. +The native file format is the `.lz4` format. + +### Difference between lz4 and gzip + +`lz4` supports a command line syntax similar _but not identical_ to `gzip(1)`. +Differences are : + + * `lz4` preserves original files + * `lz4` compresses a single file by default (see `-m` for multiple files) + * `lz4 file1 file2` means : compress file1 _into_ file2 + * `lz4 file.lz4` will default to decompression (use `-z` to force compression) + * `lz4` shows real-time notification statistics + during compression or decompression of a single file + (use `-q` to silence them) + * If no destination name is provided, result is sent to `stdout` + _except if stdout is the console_. + * If no destination name is provided, __and__ if `stdout` is the console, + `file` is compressed into `file.lz4`. + * As a consequence of previous rules, note the following example : + `lz4 file | consumer` sends compressed data to `consumer` through `stdout`, + hence it does _not_ create `file.lz4`. + * Another consequence of those rules is that to run `lz4` under `nohup`, + you should provide a destination file: `nohup lz4 file file.lz4`, + because `nohup` writes the specified command's output to a file. + +Default behaviors can be modified by opt-in commands, detailed below. + + * `lz4 -m` makes it possible to provide multiple input filenames, + which will be compressed into files using suffix `.lz4`. + Progress notifications are also disabled by default (use `-v` to enable them). + This mode has a behavior which more closely mimics `gzip` command line, + with the main remaining difference being that source files are preserved by default. + * Similarly, `lz4 -m -d` can decompress multiple `*.lz4` files. + * It's possible to opt-in to erase source files + on successful compression or decompression, using `--rm` command. + * Consequently, `lz4 -m --rm` behaves the same as `gzip`. + +### Concatenation of .lz4 files + +It is possible to concatenate `.lz4` files as is. +`lz4` will decompress such files as if they were a single `.lz4` file. +For example: + + lz4 file1 > foo.lz4 + lz4 file2 >> foo.lz4 + +Then `lz4cat foo.lz4` is equivalent to `cat file1 file2`. + +OPTIONS +------- + +### Short commands concatenation + +In some cases, some options can be expressed using short command `-x` +or long command `--long-word`. +Short commands can be concatenated together. +For example, `-d -c` is equivalent to `-dc`. +Long commands cannot be concatenated. +They must be clearly separated by a space. + +### Multiple commands + +When multiple contradictory commands are issued on a same command line, +only the latest one will be applied. + +### Operation mode + +* `-z` `--compress`: + Compress. + This is the default operation mode when no operation mode option is + specified, no other operation mode is implied from the command name + (for example, `unlz4` implies `--decompress`), + nor from the input file name + (for example, a file extension `.lz4` implies `--decompress` by default). + `-z` can also be used to force compression of an already compressed + `.lz4` file. + +* `-d` `--decompress` `--uncompress`: + Decompress. + `--decompress` is also the default operation when the input filename has an + `.lz4` extension. + +* `-t` `--test`: + Test the integrity of compressed `.lz4` files. + The decompressed data is discarded. + No files are created nor removed. + +* `-b#`: + Benchmark mode, using `#` compression level. + +### Operation modifiers + +* `-#`: + Compression level, with # being any value from 1 to 16. + Higher values trade compression speed for compression ratio. + Values above 16 are considered the same as 16. + Recommended values are 1 for fast compression (default), + and 9 for high compression. + Speed/compression trade-off will vary depending on data to compress. + Decompression speed remains fast at all settings. + +* `-f` `--[no-]force`: + This option has several effects: + + If the target file already exists, overwrite it without prompting. + + When used with `--decompress` and `lz4` cannot recognize the type of + the source file, copy the source file as is to standard output. + This allows `lz4cat --force` to be used like `cat (1)` for files + that have not been compressed with `lz4`. + +* `-c` `--stdout` `--to-stdout`: + Force write to standard output, even if it is the console. + +* `-m` `--multiple`: + Multiple input files. + Compressed file names will be appended a `.lz4` suffix. + This mode also reduces notification level. + `lz4 -m` has a behavior equivalent to `gzip -k` + (it preserves source files by default). + +* `-r` : + operate recursively on directories. + This mode also sets `-m` (multiple input files). + +* `-B#`: + Block size \[4-7\](default : 7)<br/> + `-B4`= 64KB ; `-B5`= 256KB ; `-B6`= 1MB ; `-B7`= 4MB + +* `-BD`: + Block Dependency (improves compression ratio on small blocks) + +* `--[no-]frame-crc`: + Select frame checksum (default:enabled) + +* `--[no-]content-size`: + Header includes original size (default:not present)<br/> + Note : this option can only be activated when the original size can be + determined, hence for a file. It won't work with unknown source size, + such as stdin or pipe. + +* `--[no-]sparse`: + Sparse mode support (default:enabled on file, disabled on stdout) + +* `-l`: + Use Legacy format (typically for Linux Kernel compression)<br/> + Note : `-l` is not compatible with `-m` (`--multiple`) nor `-r` + +### Other options + +* `-v` `--verbose`: + Verbose mode + +* `-q` `--quiet`: + Suppress warnings and real-time statistics; + specify twice to suppress errors too + +* `-h` `-H` `--help`: + Display help/long help and exit + +* `-V` `--version`: + Display Version number and exit + +* `-k` `--keep`: + Preserve source files (default behavior) + +* `--rm` : + Delete source files on successful compression or decompression + +* `--` : + Treat all subsequent arguments as files + + +### Benchmark mode + +* `-b#`: + Benchmark file(s), using # compression level + +* `-e#`: + Benchmark multiple compression levels, from b# to e# (included) + +* `-i#`: + Minimum evaluation in seconds \[1-9\] (default : 3) + + +BUGS +---- + +Report bugs at: https://github.com/lz4/lz4/issues + + +AUTHOR +------ + +Yann Collet diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 71378aa..362ba49 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -30,58 +30,32 @@ The license of this compression CLI program is GPLv2. */ -/************************************** -* Tuning parameters -***************************************/ -/* ENABLE_LZ4C_LEGACY_OPTIONS : - Control the availability of -c0, -c1 and -hc legacy arguments - Default : Legacy options are disabled */ -/* #define ENABLE_LZ4C_LEGACY_OPTIONS */ - - -/************************************** -* Compiler Options -***************************************/ -/* cf. http://man7.org/linux/man-pages/man7/feature_test_macros.7.html */ -#define _XOPEN_VERSION 600 /* POSIX.2001, for fileno() within <stdio.h> on unix */ - /**************************** * Includes *****************************/ -#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ +#include "platform.h" /* Compiler options, IS_CONSOLE */ +#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ #include <stdio.h> /* fprintf, getchar */ #include <stdlib.h> /* exit, calloc, free */ #include <string.h> /* strcmp, strlen */ #include "bench.h" /* BMK_benchFile, BMK_SetNbIterations, BMK_SetBlocksize, BMK_SetPause */ #include "lz4io.h" /* LZ4IO_compressFilename, LZ4IO_decompressFilename, LZ4IO_compressMultipleFilenames */ -#include "lz4hc.h" /* LZ4HC_DEFAULT_CLEVEL */ +#include "lz4hc.h" /* LZ4HC_CLEVEL_MAX */ #include "lz4.h" /* LZ4_VERSION_STRING */ -/*-************************************ -* OS-specific Includes -**************************************/ -#if defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || defined(__DJGPP__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ -# include <unistd.h> /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include <io.h> /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -# define IS_CONSOLE(stdStream) 0 -#endif - - /***************************** * Constants ******************************/ #define COMPRESSOR_NAME "LZ4 command line interface" #define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION_STRING, AUTHOR +#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_versionString(), AUTHOR #define LZ4_EXTENSION ".lz4" #define LZ4CAT "lz4cat" #define UNLZ4 "unlz4" +#define LZ4_LEGACY "lz4c" +static int g_lz4c_legacy_commands = 0; #define KB *(1U<<10) #define MB *(1U<<20) @@ -116,9 +90,6 @@ static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : dow /*-************************************ * Version modifiers ***************************************/ -#define EXTENDED_ARGUMENTS -#define EXTENDED_HELP -#define EXTENDED_FORMAT #define DEFAULT_COMPRESSOR LZ4IO_compressFilename #define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */ @@ -129,19 +100,21 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output *****************************/ static int usage(const char* exeName) { - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] [input] [output]\n", exeName); + DISPLAY( "Usage : \n"); + DISPLAY( " %s [arg] [input] [output] \n", exeName); DISPLAY( "\n"); - DISPLAY( "input : a filename\n"); + DISPLAY( "input : a filename \n"); DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); - DISPLAY( "Arguments :\n"); + DISPLAY( "Arguments : \n"); DISPLAY( " -1 : Fast compression (default) \n"); DISPLAY( " -9 : High compression \n"); DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION); - DISPLAY( " -z : force compression\n"); + DISPLAY( " -z : force compression \n"); + DISPLAY( " -D FILE: use dictionary in FILE \n"); DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( " -k : preserve source files(s) (default) \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); - DISPLAY( " -h/-H : display help/long help and exit\n"); + DISPLAY( " -h/-H : display help/long help and exit \n"); return 0; } @@ -151,36 +124,35 @@ static int usage_advanced(const char* exeName) usage(exeName); DISPLAY( "\n"); DISPLAY( "Advanced arguments :\n"); - DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( " -v : verbose mode\n"); + DISPLAY( " -V : display Version number and exit \n"); + DISPLAY( " -v : verbose mode \n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -t : test compressed file integrity\n"); DISPLAY( " -m : multiple input files (implies automatic output filenames)\n"); #ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories (sets also -m)\n"); + DISPLAY( " -r : operate recursively on directories (sets also -m) \n"); #endif DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n"); - DISPLAY( " -B# : Block size [4-7] (default : 7)\n"); - DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); - /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */ - DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n"); + DISPLAY( " -B# : Block size [4-7] (default : 7) \n"); + DISPLAY( " -BD : Block dependency (improve compression ratio) \n"); + DISPLAY( " -BX : enable block checksum (default:disabled) \n"); + DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n"); DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); - DISPLAY( "Benchmark arguments :\n"); + DISPLAY( "Benchmark arguments : \n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); - DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); - DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+]\n"); - DISPLAY( " or predefined block size [4-7] (default: 7)\n"); -#if defined(ENABLE_LZ4C_LEGACY_OPTIONS) - DISPLAY( "Legacy arguments :\n"); - DISPLAY( " -c0 : fast compression\n"); - DISPLAY( " -c1 : high compression\n"); - DISPLAY( " -hc : high compression\n"); - DISPLAY( " -y : overwrite output without prompting \n"); -#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */ - EXTENDED_HELP; + DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n"); + DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+] \n"); + DISPLAY( " or predefined block size [4-7] (default: 7) \n"); + if (g_lz4c_legacy_commands) { + DISPLAY( "Legacy arguments : \n"); + DISPLAY( " -c0 : fast compression \n"); + DISPLAY( " -c1 : high compression \n"); + DISPLAY( " -c2,-hc: very high compression \n"); + DISPLAY( " -y : overwrite output without prompting \n"); + } return 0; } @@ -207,7 +179,7 @@ static int usage_longhelp(const char* exeName) DISPLAY( "Compression levels : \n"); DISPLAY( "---------------------\n"); DISPLAY( "-0 ... -2 => Fast compression, all identicals\n"); - DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_MAX_CLEVEL); + DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX); DISPLAY( "\n"); DISPLAY( "stdin, stdout and the console : \n"); DISPLAY( "--------------------------------\n"); @@ -231,17 +203,17 @@ static int usage_longhelp(const char* exeName) DISPLAY( "-------------------------------------\n"); DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n"); DISPLAY( " generator | %s | consumer \n", exeName); -#if defined(ENABLE_LZ4C_LEGACY_OPTIONS) - DISPLAY( "\n"); - DISPLAY( "***** Warning *****\n"); - DISPLAY( "Legacy arguments take precedence. Therefore : \n"); - DISPLAY( "---------------------------------\n"); - DISPLAY( " %s -hc filename\n", exeName); - DISPLAY( "means 'compress filename in high compression mode'\n"); - DISPLAY( "It is not equivalent to :\n"); - DISPLAY( " %s -h -c filename\n", exeName); - DISPLAY( "which would display help text and exit\n"); -#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */ + if (g_lz4c_legacy_commands) { + DISPLAY( "\n"); + DISPLAY( "***** Warning ***** \n"); + DISPLAY( "Legacy arguments take precedence. Therefore : \n"); + DISPLAY( "--------------------------------- \n"); + DISPLAY( " %s -hc filename \n", exeName); + DISPLAY( "means 'compress filename in high compression mode' \n"); + DISPLAY( "It is not equivalent to : \n"); + DISPLAY( " %s -h -c filename \n", exeName); + DISPLAY( "which displays help text and exits \n"); + } return 0; } @@ -259,16 +231,43 @@ static void waitEnter(void) (void)getchar(); } +static const char* lastNameFromPath(const char* path) +{ + const char* name = path; + if (strrchr(name, '/')) name = strrchr(name, '/') + 1; + if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */ + return name; +} + +/*! exeNameMatch() : + @return : a non-zero value if exeName matches test, excluding the extension + */ +static int exeNameMatch(const char* exeName, const char* test) +{ + return !strncmp(exeName, test, strlen(test)) && + (exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.'); +} /*! readU32FromChar() : - @return : unsigned integer value reach from input in `char` format + @return : unsigned integer value read from input in `char` format + allows and interprets K, KB, KiB, M, MB and MiB suffix. Will also modify `*stringPtr`, advancing it to position where it stopped reading. - Note : this function can overflow if result > MAX_UINT */ + Note : function result can overflow if digit string > MAX_UINT */ static unsigned readU32FromChar(const char** stringPtr) { unsigned result = 0; - while ((**stringPtr >='0') && (**stringPtr <='9')) - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + result *= 10; + result += **stringPtr - '0'; + (*stringPtr)++ ; + } + if ((**stringPtr=='K') || (**stringPtr=='M')) { + result <<= 10; + if (**stringPtr=='M') result <<= 10; + (*stringPtr)++ ; + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } return result; } @@ -283,17 +282,19 @@ int main(int argc, const char** argv) forceStdout=0, main_pause=0, multiple_inputs=0, + all_arguments_are_files=0, operationResult=0; operationMode_e mode = om_auto; const char* input_filename = NULL; const char* output_filename= NULL; + const char* dictionary_filename = NULL; char* dynNameSpace = NULL; const char** inFileNames = (const char**) calloc(argc, sizeof(char*)); unsigned ifnIdx=0; const char nullOutput[] = NULL_OUTPUT; const char extension[] = LZ4_EXTENSION; size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); - const char* const exeName = argv[0]; + const char* const exeName = lastNameFromPath(argv[0]); #ifdef UTIL_HAS_CREATEFILELIST const char** extendedFileList = NULL; char* fileNamesBuf = NULL; @@ -305,18 +306,21 @@ int main(int argc, const char** argv) DISPLAY("Allocation error : not enough memory \n"); return 1; } + inFileNames[0] = stdinmark; LZ4IO_setOverwrite(0); - /* lz4cat predefined behavior */ - if (!strcmp(exeName, LZ4CAT)) { + /* predefined behaviors, based on binary/link name */ + if (exeNameMatch(exeName, LZ4CAT)) { mode = om_decompress; LZ4IO_setOverwrite(1); + LZ4IO_setRemoveSrcFile(0); forceStdout=1; output_filename=stdoutmark; displayLevel=1; multiple_inputs=1; } - if (!strcmp(exeName, UNLZ4)) { mode = om_decompress; } + if (exeNameMatch(exeName, UNLZ4)) { mode = om_decompress; } + if (exeNameMatch(exeName, LZ4_LEGACY)) { g_lz4c_legacy_commands=1; } /* command switches */ for(i=1; i<argc; i++) { @@ -325,7 +329,7 @@ int main(int argc, const char** argv) if(!argument) continue; /* Protection if argument empty */ /* Short commands (note : aggregated short commands are allowed) */ - if (argument[0]=='-') { + if (!all_arguments_are_files && argument[0]=='-') { /* '-' means stdin/stdout */ if (argument[1]==0) { if (!input_filename) input_filename=stdinmark; @@ -335,6 +339,7 @@ int main(int argc, const char** argv) /* long commands (--long-word) */ if (argument[1]=='-') { + if (!strcmp(argument, "--")) { all_arguments_are_files = 1; continue; } if (!strcmp(argument, "--compress")) { mode = om_compress; continue; } if ((!strcmp(argument, "--decompress")) || (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; } @@ -361,13 +366,14 @@ int main(int argc, const char** argv) while (argument[1]!=0) { argument ++; -#if defined(ENABLE_LZ4C_LEGACY_OPTIONS) - /* Legacy arguments (-c0, -c1, -hc, -y, -s) */ - if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } /* -c0 (fast compression) */ - if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } /* -c1 (high compression) */ - if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } /* -hc (high compression) */ - if (*argument=='y') { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */ -#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */ + if (g_lz4c_legacy_commands) { + /* Legacy commands (-c0, -c1, -hc, -y) */ + if (!strcmp(argument, "c0")) { cLevel=0; argument++; continue; } /* -c0 (fast compression) */ + if (!strcmp(argument, "c1")) { cLevel=9; argument++; continue; } /* -c1 (high compression) */ + if (!strcmp(argument, "c2")) { cLevel=12; argument++; continue; } /* -c2 (very high compression) */ + if (!strcmp(argument, "hc")) { cLevel=12; argument++; continue; } /* -hc (very high compression) */ + if (!strcmp(argument, "y")) { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */ + } if ((*argument>='0') && (*argument<='9')) { cLevel = readU32FromChar(&argument); @@ -392,6 +398,22 @@ int main(int argc, const char** argv) /* Compression (default) */ case 'z': mode = om_compress; break; + case 'D': + if (argument[1] == '\0') { + /* path is next arg */ + if (i + 1 == argc) { + /* there is no next arg */ + badusage(exeName); + } + dictionary_filename = argv[++i]; + } else { + /* path follows immediately */ + dictionary_filename = argument + 1; + } + /* skip to end of argument so that we jump to parsing next argument */ + argument += strlen(argument) - 1; + break; + /* Use Legacy format (ex : Linux kernel compression) */ case 'l': legacy_format = 1; blockSize = 8 MB; break; @@ -459,9 +481,10 @@ int main(int argc, const char** argv) break; #ifdef UTIL_HAS_CREATEFILELIST - /* recursive */ - case 'r': recursive=1; /* without break */ + /* recursive */ + case 'r': recursive=1; #endif + /* fall-through */ /* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */ case 'm': multiple_inputs=1; break; @@ -473,16 +496,13 @@ int main(int argc, const char** argv) iters = readU32FromChar(&argument); argument--; BMK_setNotificationLevel(displayLevel); - BMK_SetNbSeconds(iters); + BMK_SetNbSeconds(iters); /* notification if displayLevel >= 3 */ } break; /* Pause at the end (hidden option) */ case 'p': main_pause=1; break; - /* Specific commands for customized versions */ - EXTENDED_ARGUMENTS; - /* Unrecognised command */ default : badusage(exeName); } @@ -508,7 +528,20 @@ int main(int argc, const char** argv) } DISPLAYLEVEL(3, WELCOME_MESSAGE); - if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %i KB\n", (U32)(blockSize>>10)); +#ifdef _POSIX_C_SOURCE + DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE); +#endif +#ifdef _POSIX_VERSION + DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION); +#endif +#ifdef PLATFORM_POSIX_VERSION + DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); +#endif +#ifdef _FILE_OFFSET_BITS + DISPLAYLEVEL(4, "_FILE_OFFSET_BITS defined: %ldL\n", (long) _FILE_OFFSET_BITS); +#endif + if ((mode == om_compress) || (mode == om_bench)) + DISPLAYLEVEL(4, "Blocks size : %u KB\n", (U32)(blockSize>>10)); if (multiple_inputs) { input_filename = inFileNames[0]; @@ -521,8 +554,7 @@ int main(int argc, const char** argv) free((void*)inFileNames); inFileNames = extendedFileList; ifnIdx = fileNamesNb; - } - } + } } #endif } @@ -539,6 +571,14 @@ int main(int argc, const char** argv) mode = om_decompress; /* defer to decompress */ } + if (dictionary_filename) { + if (!strcmp(dictionary_filename, stdinmark) && IS_CONSOLE(stdin)) { + DISPLAYLEVEL(1, "refusing to read from a console\n"); + exit(1); + } + LZ4IO_setDictionaryFilename(dictionary_filename); + } + /* compress or decompress */ if (!input_filename) input_filename = stdinmark; /* Check if input is defined as console; trigger an error in this case */ @@ -546,9 +586,12 @@ int main(int argc, const char** argv) DISPLAYLEVEL(1, "refusing to read from a console\n"); exit(1); } + /* if input==stdin and no output defined, stdout becomes default output */ + if (!strcmp(input_filename, stdinmark) && !output_filename) + output_filename = stdoutmark; /* No output filename ==> try to select one automatically (when possible) */ - while (!output_filename) { + while ((!output_filename) && (multiple_inputs==0)) { if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ size_t const inSize = strlen(input_filename); @@ -584,17 +627,18 @@ int main(int argc, const char** argv) } /* Check if output is defined as console; trigger an error in this case */ + if (!output_filename) output_filename = "*\\dummy^!//"; if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) { DISPLAYLEVEL(1, "refusing to write to console without -c\n"); exit(1); } - /* Downgrade notification level in stdout and multiple file mode */ if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; /* IO Stream/File */ LZ4IO_setNotificationLevel(displayLevel); + if (ifnIdx == 0) multiple_inputs = 0; if (mode == om_decompress) { if (multiple_inputs) operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION); @@ -602,7 +646,7 @@ int main(int argc, const char** argv) operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename); } else { /* compression is default action */ if (legacy_format) { - DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n"); + DISPLAYLEVEL(3, "! Generating LZ4 Legacy format (deprecated) ! \n"); LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); } else { if (multiple_inputs) @@ -614,12 +658,13 @@ int main(int argc, const char** argv) _cleanup: if (main_pause) waitEnter(); - if (dynNameSpace) free(dynNameSpace); + free(dynNameSpace); #ifdef UTIL_HAS_CREATEFILELIST - if (extendedFileList) + if (extendedFileList) { UTIL_freeFileList(extendedFileList, fileNamesBuf); - else + inFileNames = NULL; + } #endif - free((void*)inFileNames); + free((void*)inFileNames); return operationResult; } diff --git a/programs/lz4io.c b/programs/lz4io.c index bc2ba95..2cf0c1c 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -1,6 +1,6 @@ /* LZ4io.c - LZ4 File/Stream Interface - Copyright (C) Yann Collet 2011-2016 + Copyright (C) Yann Collet 2011-2017 GPL v2 License @@ -30,24 +30,26 @@ - The license of this source file is GPLv2. */ -/************************************** -* Compiler Options -**************************************/ -#define _LARGE_FILES /* Large file support on 32-bits AIX */ -#define _FILE_OFFSET_BITS 64 /* off_t width */ -#define _LARGEFILE_SOURCE +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ #endif + /***************************** * Includes *****************************/ -#include "util.h" /* Compiler options, UTIL_getFileStat */ +#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE, PLATFORM_POSIX_VERSION, __64BIT__ */ +#include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ #include <stdio.h> /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include <stdlib.h> /* malloc, free */ -#include <string.h> /* strcmp, strlen */ +#include <string.h> /* strerror, strcmp, strlen */ #include <time.h> /* clock */ #include <sys/types.h> /* stat64 */ #include <sys/stat.h> /* stat64 */ @@ -55,33 +57,7 @@ #include "lz4.h" /* still required for legacy format */ #include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" - - -/****************************** -* OS-specific Includes -******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) -# include <fcntl.h> /* _O_BINARY */ -# include <io.h> /* _setmode, _fileno, _get_osfhandle */ -# if !defined(__DJGPP__) -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } -# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include <winioctl.h> /* FSCTL_SET_SPARSE */ -# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } -# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ -# define fseek _fseeki64 -# endif -# else -# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) -# define SET_SPARSE_FILE_MODE(file) -# endif -#else -# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__)) -# define fseek fseeko -# endif -# define SET_BINARY_MODE(file) -# define SET_SPARSE_FILE_MODE(file) -#endif +#include "lz4frame_static.h" /***************************** @@ -107,6 +83,7 @@ #define LEGACY_BLOCKSIZE (8 MB) #define MIN_STREAM_BUFSIZE (192 KB) #define LZ4IO_BLOCKSIZEID_DEFAULT 7 +#define LZ4_MAX_DICT_SIZE (64 KB) /************************************** @@ -135,6 +112,8 @@ static int g_streamChecksum = 1; static int g_blockIndependence = 1; static int g_sparseFileSupport = 1; static int g_contentSizeFlag = 0; +static int g_useDictionary = 0; +static const char* g_dictionaryFilename = NULL; /************************************** @@ -167,6 +146,12 @@ static int g_contentSizeFlag = 0; /* ****************** Parameters ******************** */ /* ************************************************** */ +int LZ4IO_setDictionaryFilename(const char* dictionaryFilename) { + g_dictionaryFilename = dictionaryFilename; + g_useDictionary = dictionaryFilename != NULL; + return g_useDictionary; +} + /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ int LZ4IO_setOverwrite(int yes) { @@ -198,17 +183,17 @@ int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) return g_blockIndependence; } -/* Default setting : no checksum */ -int LZ4IO_setBlockChecksumMode(int xxhash) +/* Default setting : no block checksum */ +int LZ4IO_setBlockChecksumMode(int enable) { - g_blockChecksum = (xxhash != 0); + g_blockChecksum = (enable != 0); return g_blockChecksum; } /* Default setting : checksum enabled */ -int LZ4IO_setStreamChecksumMode(int xxhash) +int LZ4IO_setStreamChecksumMode(int enable) { - g_streamChecksum = (xxhash != 0); + g_streamChecksum = (enable != 0); return g_streamChecksum; } @@ -243,11 +228,13 @@ void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } ** ************************************************************************ */ static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } -static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; } +static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { + return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; +} /** LZ4IO_openSrcFile() : - * condition : `dstFileName` must be non-NULL. + * condition : `srcFileName` must be non-NULL. * @result : FILE* to `dstFileName`, or NULL if it fails */ static FILE* LZ4IO_openSrcFile(const char* srcFileName) { @@ -316,7 +303,7 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName) /* unoptimized version; solves endianess & alignment issues */ static void LZ4IO_writeLE32 (void* p, unsigned value32) { - unsigned char* dstPtr = (unsigned char*)p; + unsigned char* const dstPtr = (unsigned char*)p; dstPtr[0] = (unsigned char)value32; dstPtr[1] = (unsigned char)(value32 >> 8); dstPtr[2] = (unsigned char)(value32 >> 16); @@ -342,11 +329,11 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE); FILE* finput; FILE* foutput; - clock_t end; + clock_t clockEnd; /* Init */ - clock_t const start = clock(); - if (compressionlevel < 3) compressionFunction = LZ4IO_LZ4_compress; else compressionFunction = LZ4_compress_HC; + clock_t const clockStart = clock(); + compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC; finput = LZ4IO_openSrcFile(input_filename); if (finput == NULL) EXM_THROW(20, "%s : open file error ", input_filename); @@ -361,7 +348,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output /* Write Archive Header */ LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER); { size_t const sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); } + if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); } /* Main Loop */ while (1) { @@ -375,24 +362,27 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output /* Compress Block */ outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel); compressedfilesize += outSize+4; - DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", + (int)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ LZ4IO_writeLE32(out_buff, outSize); { size_t const sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); - if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(24, "Write error : cannot write compressed block"); + if (sizeCheck!=(size_t)(outSize+4)) + EXM_THROW(24, "Write error : cannot write compressed block"); } } if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename); /* Status */ - end = clock(); - if (end==start) end+=1; /* avoid division by zero (speed) */ + clockEnd = clock(); + if (clockEnd==clockStart) clockEnd+=1; /* avoid division by zero (speed) */ filesize += !filesize; /* avoid division by zero (ratio) */ DISPLAYLEVEL(2, "\r%79s\r", ""); /* blank line */ DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", filesize, compressedfilesize, (double)compressedfilesize / filesize * 100); - { double const seconds = (double)(end - start) / CLOCKS_PER_SEC; - DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + { double const seconds = (double)(clockEnd - clockStart) / CLOCKS_PER_SEC; + DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, + (double)filesize / seconds / 1024 / 1024); } /* Close & Free */ @@ -415,8 +405,79 @@ typedef struct { void* dstBuffer; size_t dstBufferSize; LZ4F_compressionContext_t ctx; + LZ4F_CDict* cdict; } cRess_t; +static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { + size_t readSize; + size_t dictEnd = 0; + size_t dictLen = 0; + size_t dictStart; + size_t circularBufSize = LZ4_MAX_DICT_SIZE; + char* circularBuf; + char* dictBuf; + FILE* dictFile; + + if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided"); + + circularBuf = (char *) malloc(circularBufSize); + if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory"); + + dictFile = LZ4IO_openSrcFile(dictFilename); + if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file"); + + /* opportunistically seek to the part of the file we care about. If this */ + /* fails it's not a problem since we'll just read everything anyways. */ + if (strcmp(dictFilename, stdinmark)) { + UTIL_fseek(dictFile, -LZ4_MAX_DICT_SIZE, SEEK_END); + } + + do { + readSize = fread(circularBuf + dictEnd, 1, circularBufSize - dictEnd, dictFile); + dictEnd = (dictEnd + readSize) % circularBufSize; + dictLen += readSize; + } while (readSize>0); + + if (dictLen > LZ4_MAX_DICT_SIZE) { + dictLen = LZ4_MAX_DICT_SIZE; + } + + *dictSize = dictLen; + + dictStart = (circularBufSize + dictEnd - dictLen) % circularBufSize; + + if (dictStart == 0) { + /* We're in the simple case where the dict starts at the beginning of our circular buffer. */ + dictBuf = circularBuf; + circularBuf = NULL; + } else { + /* Otherwise, we will alloc a new buffer and copy our dict into that. */ + dictBuf = (char *) malloc(dictLen ? dictLen : 1); + if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory"); + + memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart); + memcpy(dictBuf + circularBufSize - dictStart, circularBuf, dictLen - (circularBufSize - dictStart)); + } + + free(circularBuf); + + return dictBuf; +} + +static LZ4F_CDict* LZ4IO_createCDict(void) { + size_t dictionarySize; + void* dictionaryBuffer; + LZ4F_CDict* cdict; + if (!g_useDictionary) { + return NULL; + } + dictionaryBuffer = LZ4IO_createDict(g_dictionaryFilename, &dictionarySize); + if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); + cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize); + free(dictionaryBuffer); + return cdict; +} + static cRess_t LZ4IO_createCResources(void) { const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); @@ -432,6 +493,8 @@ static cRess_t LZ4IO_createCResources(void) ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); + ress.cdict = LZ4IO_createCDict(); + return ress; } @@ -439,6 +502,10 @@ static void LZ4IO_freeCResources(cRess_t ress) { free(ress.srcBuffer); free(ress.dstBuffer); + + LZ4F_freeCDict(ress.cdict); + ress.cdict = NULL; + { LZ4F_errorCode_t const errorCode = LZ4F_freeCompressionContext(ress.ctx); if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); } } @@ -475,6 +542,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, prefs.compressionLevel = compressionLevel; prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence; prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId; + prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum; prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum; if (g_contentSizeFlag) { U64 const fileSize = UTIL_getFileSize(srcFileName); @@ -491,7 +559,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* single-block file */ if (readSize < blockSize) { /* Compress in single pass */ - size_t const cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs); + size_t cSize = LZ4F_compressFrame_usingCDict(dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs); if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize)); compressedfilesize = cSize; DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", @@ -507,7 +575,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* multiple-blocks file */ { /* Write Archive Header */ - size_t headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); + size_t headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs); if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } @@ -548,7 +616,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* Copy owner, file permissions and modification time */ { stat_t statbuf; - if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && UTIL_getFileStat(srcFileName, &statbuf)) + if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && strcmp (dstFileName, nulmark) && UTIL_getFileStat(srcFileName, &statbuf)) UTIL_setFileStat(dstFileName, &statbuf); } @@ -591,12 +659,15 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, char* dstFileName = (char*)malloc(FNSPACE); size_t ofnSize = FNSPACE; const size_t suffixSize = strlen(suffix); - cRess_t const ress = LZ4IO_createCResources(); + cRess_t ress; + + if (dstFileName == NULL) return ifntSize; /* not enough memory */ + ress = LZ4IO_createCResources(); /* loop on each file */ for (i=0; i<ifntSize; i++) { size_t const ifnSize = strlen(inFileNamesTable[i]); - if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); } + if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); if (dstFileName==NULL) { LZ4IO_freeCResources(ress); return ifntSize; } } strcpy(dstFileName, inFileNamesTable[i]); strcat(dstFileName, suffix); @@ -625,16 +696,16 @@ static unsigned LZ4IO_readLE32 (const void* s) return value32; } -#define sizeT sizeof(size_t) -#define maskT (sizeT - 1) static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) { + const size_t sizeT = sizeof(size_t); + const size_t maskT = sizeT -1 ; const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ const size_t* ptrT = bufferT; size_t bufferSizeT = bufferSize / sizeT; const size_t* const bufferTEnd = bufferT + bufferSizeT; - static const size_t segmentSizeT = (32 KB) / sizeT; + const size_t segmentSizeT = (32 KB) / sizeT; if (!g_sparseFileSupport) { /* normal write */ size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); @@ -644,7 +715,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer /* avoid int overflow */ if (storedSkips > 1 GB) { - int const seekResult = fseek(file, 1 GB, SEEK_CUR); + int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR); if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); storedSkips -= 1 GB; } @@ -660,8 +731,10 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer storedSkips += (unsigned)(nb0T * sizeT); if (nb0T != seg0SizeT) { /* not all 0s */ - int const seekResult = fseek(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); + errno = 0; + { int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno)); + } storedSkips = 0; seg0SizeT -= nb0T; ptrT += nb0T; @@ -679,7 +752,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; storedSkips += (unsigned) (restPtr - restStart); if (restPtr != restEnd) { - int const seekResult = fseek(file, storedSkips, SEEK_CUR); + int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); storedSkips = 0; { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); @@ -693,7 +766,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) { if (storedSkips>0) { /* implies g_sparseFileSupport>0 */ - int const seekResult = fseek(file, storedSkips-1, SEEK_CUR); + int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR); if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); { const char lastZeroByte[1] = { 0 }; size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); @@ -702,22 +775,19 @@ static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) } -static unsigned g_magicRead = 0; +static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) { - unsigned long long filesize = 0; - char* in_buff; - char* out_buff; + unsigned long long streamSize = 0; unsigned storedSkips = 0; /* Allocate Memory */ - in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); - out_buff = (char*)malloc(LEGACY_BLOCKSIZE); + char* const in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); + char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE); if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); /* Main Loop */ while (1) { - int decodeSize; unsigned int blockSize; /* Block Size */ @@ -736,13 +806,12 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); } /* Decode Block */ - decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); - if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); - filesize += decodeSize; - - /* Write Block */ - storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); - } + { int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); + if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); + streamSize += decodeSize; + /* Write Block */ + storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); /* success or die */ + } } if (ferror(finput)) EXM_THROW(54, "Read error : ferror"); LZ4IO_fwriteSparseEnd(foutput, storedSkips); @@ -751,7 +820,7 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) free(in_buff); free(out_buff); - return filesize; + return streamSize; } @@ -763,8 +832,21 @@ typedef struct { size_t dstBufferSize; FILE* dstFile; LZ4F_decompressionContext_t dCtx; + void* dictBuffer; + size_t dictBufferSize; } dRess_t; +static void LZ4IO_loadDDict(dRess_t* ress) { + if (!g_useDictionary) { + ress->dictBuffer = NULL; + ress->dictBufferSize = 0; + return; + } + + ress->dictBuffer = LZ4IO_createDict(g_dictionaryFilename, &ress->dictBufferSize); + if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); +} + static const size_t LZ4IO_dBufferSize = 64 KB; static dRess_t LZ4IO_createDResources(void) { @@ -781,6 +863,8 @@ static dRess_t LZ4IO_createDResources(void) ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); + LZ4IO_loadDDict(&ress); + ress.dstFile = NULL; return ress; } @@ -791,6 +875,7 @@ static void LZ4IO_freeDResources(dRess_t ress) if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); free(ress.srcBuffer); free(ress.dstBuffer); + free(ress.dictBuffer); } @@ -804,7 +889,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE { size_t inSize = MAGICNUMBER_SIZE; size_t outSize= 0; LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER); - nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL); + nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, ress.dictBuffer, ress.dictBufferSize, NULL); if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad)); } @@ -823,7 +908,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE /* Decode Input (at least partially) */ size_t remaining = readSize - pos; decodedBytes = ress.dstBufferSize; - nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); + nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, ress.dictBuffer, ress.dictBufferSize, NULL); if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); pos += remaining; @@ -882,7 +967,7 @@ static int fseek_u32(FILE *fp, unsigned offset, int where) while (offset > 0) { unsigned s = offset; if (s > stepMax) s = stepMax; - errorNb = fseek(fp, (long) s, SEEK_CUR); + errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR); if (errorNb != 0) break; offset -= s; } @@ -894,22 +979,24 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu { unsigned char MNstore[MAGICNUMBER_SIZE]; unsigned magicNumber; - static unsigned nbCalls = 0; + static unsigned nbFrames = 0; /* init */ - nbCalls++; + nbFrames++; /* Check Archive Header */ if (g_magicRead) { /* magic number already read from finput (see legacy frame)*/ - magicNumber = g_magicRead; - g_magicRead = 0; + magicNumber = g_magicRead; + g_magicRead = 0; } else { - size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); - if (nbReadBytes==0) { nbCalls = 0; return ENDOFSTREAM; } /* EOF */ - if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); - magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */ + size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); + if (nbReadBytes==0) { nbFrames = 0; return ENDOFSTREAM; } /* EOF */ + if (nbReadBytes != MAGICNUMBER_SIZE) + EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */ } - if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ + if (LZ4IO_isSkippableMagicNumber(magicNumber)) + magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ switch(magicNumber) { @@ -920,22 +1007,32 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu return LZ4IO_decodeLegacyStream(finput, foutput); case LZ4IO_SKIPPABLE0: DISPLAYLEVEL(4, "Skipping detected skippable area \n"); - { size_t const nbReadBytes = fread(MNstore, 1, 4, finput); - if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); } - { unsigned const size = LZ4IO_readLE32(MNstore); /* Little Endian format */ - int const errorNb = fseek_u32(finput, size, SEEK_CUR); - if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); } + { size_t const nbReadBytes = fread(MNstore, 1, 4, finput); + if (nbReadBytes != 4) + EXM_THROW(42, "Stream error : skippable size unreadable"); + } + { unsigned const size = LZ4IO_readLE32(MNstore); + int const errorNb = fseek_u32(finput, size, SEEK_CUR); + if (errorNb != 0) + EXM_THROW(43, "Stream error : cannot skip skippable area"); + } return 0; EXTENDED_FORMAT; /* macro extension for custom formats */ default: - if (nbCalls == 1) { /* just started */ + if (nbFrames == 1) { /* just started */ + /* Wrong magic number at the beginning of 1st stream */ if (!g_testMode && g_overwrite) { - nbCalls = 0; + nbFrames = 0; return LZ4IO_passThrough(finput, foutput, MNstore); } - EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ + EXM_THROW(44,"Unrecognized header : file cannot be decoded"); + } + { long int const position = ftell(finput); /* only works for files < 2 GB */ + DISPLAYLEVEL(2, "Stream followed by undecodable data "); + if (position != -1L) + DISPLAYLEVEL(2, "at position %i ", (int)position); + DISPLAYLEVEL(2, "\n"); } - DISPLAYLEVEL(2, "Stream followed by undecodable data\n"); return ENDOFSTREAM; } } @@ -944,24 +1041,26 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename) { FILE* const foutput = ress.dstFile; - unsigned long long filesize = 0, decodedSize=0; - FILE* finput; + unsigned long long filesize = 0; /* Init */ - finput = LZ4IO_openSrcFile(input_filename); + FILE* const finput = LZ4IO_openSrcFile(input_filename); if (finput==NULL) return 1; /* Loop over multiple streams */ - do { - decodedSize = selectDecoder(ress, finput, foutput); - if (decodedSize != ENDOFSTREAM) - filesize += decodedSize; - } while (decodedSize != ENDOFSTREAM); + for ( ; ; ) { /* endless loop, see break condition */ + unsigned long long const decodedSize = + selectDecoder(ress, finput, foutput); + if (decodedSize == ENDOFSTREAM) break; + filesize += decodedSize; + } - /* Close */ + /* Close input */ fclose(finput); - - if (g_removeSrcFile) { if (remove(input_filename)) EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); } /* remove source file : --rm */ + if (g_removeSrcFile) { /* --rm */ + if (remove(input_filename)) + EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); + } /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); @@ -974,21 +1073,26 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename) { - FILE* foutput; - - /* Init */ - foutput = LZ4IO_openDstFile(output_filename); + stat_t statbuf; + int stat_result = 0; + FILE* const foutput = LZ4IO_openDstFile(output_filename); if (foutput==NULL) return 1; /* failure */ + if ( strcmp(input_filename, stdinmark) + && UTIL_getFileStat(input_filename, &statbuf)) + stat_result = 1; + ress.dstFile = foutput; LZ4IO_decompressSrcFile(ress, input_filename, output_filename); fclose(foutput); /* Copy owner, file permissions and modification time */ - { stat_t statbuf; - if (strcmp (input_filename, stdinmark) && strcmp (output_filename, stdoutmark) && UTIL_getFileStat(input_filename, &statbuf)) - UTIL_setFileStat(output_filename, &statbuf); + if ( stat_result != 0 + && strcmp (output_filename, stdoutmark) + && strcmp (output_filename, nulmark)) { + UTIL_setFileStat(output_filename, &statbuf); + /* should return value be read ? or is silent fail good enough ? */ } return 0; @@ -1002,10 +1106,9 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename); - { clock_t const end = clock(); - double const seconds = (double)(end - start) / CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds); - } + clock_t const end = clock(); + double const seconds = (double)(end - start) / CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds); LZ4IO_freeDResources(ress); return missingFiles; @@ -1022,7 +1125,7 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz size_t const suffixSize = strlen(suffix); dRess_t ress = LZ4IO_createDResources(); - if (outFileName==NULL) exit(1); /* not enough memory */ + if (outFileName==NULL) return ifntSize; /* not enough memory */ ress.dstFile = LZ4IO_openDstFile(stdoutmark); for (i=0; i<ifntSize; i++) { @@ -1032,7 +1135,7 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark); continue; } - if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) exit(1); } + if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) return ifntSize; } if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) { DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); skippedFiles++; diff --git a/programs/lz4io.h b/programs/lz4io.h index 6190f00..b21b8b6 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -64,6 +64,8 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz /* ****************** Parameters ******************** */ /* ************************************************** */ +int LZ4IO_setDictionaryFilename(const char* dictionaryFilename); + /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ int LZ4IO_setOverwrite(int yes); diff --git a/programs/platform.h b/programs/platform.h new file mode 100644 index 0000000..db2efac --- /dev/null +++ b/programs/platform.h @@ -0,0 +1,154 @@ +/* + platform.h - compiler and OS detection + Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#ifndef PLATFORM_H_MODULE +#define PLATFORM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + + +/* ************************************** +* Compiler Options +****************************************/ +#if defined(_MSC_VER) +# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ +# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */ +# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ +# endif +#endif + + +/* ************************************** +* Detect 64-bit OS +* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros +****************************************/ +#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ + || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ + || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ + || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ + || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ + || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ + || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ +# if !defined(__64BIT__) +# define __64BIT__ 1 +# endif +#endif + + +/* ********************************************************* +* Turn on Large Files support (>4GB) for 32-bit Linux/Unix +***********************************************************/ +#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */ +# if !defined(_FILE_OFFSET_BITS) +# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ +# endif +# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ +# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ +# endif +# if defined(_AIX) || defined(__hpux) +# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ +# endif +#endif + + +/* ************************************************************ +* Detect POSIX version +* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows +* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX +* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION +***************************************************************/ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ + || defined(__midipix__) || defined(__VMS)) +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \ + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ +# define PLATFORM_POSIX_VERSION 200112L +# else +# if defined(__linux__) || defined(__linux) +# ifndef _POSIX_C_SOURCE +# define _POSIX_C_SOURCE 200112L /* use feature test macro */ +# endif +# endif +# include <unistd.h> /* declares _POSIX_VERSION */ +# if defined(_POSIX_VERSION) /* POSIX compliant */ +# define PLATFORM_POSIX_VERSION _POSIX_VERSION +# else +# define PLATFORM_POSIX_VERSION 0 +# endif +# endif +#endif +#if !defined(PLATFORM_POSIX_VERSION) +# define PLATFORM_POSIX_VERSION -1 +#endif + + +/*-********************************************* +* Detect if isatty() and fileno() are available +************************************************/ +#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) +# include <unistd.h> /* isatty */ +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#elif defined(MSDOS) || defined(OS2) || defined(__CYGWIN__) +# include <io.h> /* _isatty */ +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#elif defined(WIN32) || defined(_WIN32) +# include <io.h> /* _isatty */ +# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include <stdio.h> /* FILE */ +static __inline int IS_CONSOLE(FILE* stdStream) +{ + DWORD dummy; + return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy); +} +#else +# define IS_CONSOLE(stdStream) 0 +#endif + + +/****************************** +* OS-specific Includes +******************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) +# include <fcntl.h> /* _O_BINARY */ +# include <io.h> /* _setmode, _fileno, _get_osfhandle */ +# if !defined(__DJGPP__) +# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include <winioctl.h> /* FSCTL_SET_SPARSE */ +# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } +# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } +# else +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +# define SET_SPARSE_FILE_MODE(file) +# endif +#else +# define SET_BINARY_MODE(file) +# define SET_SPARSE_FILE_MODE(file) +#endif + + + +#if defined (__cplusplus) +} +#endif + +#endif /* PLATFORM_H_MODULE */ diff --git a/programs/util.h b/programs/util.h index 1ad61bc..fc7f63e 100644 --- a/programs/util.h +++ b/programs/util.h @@ -15,11 +15,8 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ + #ifndef UTIL_H_MODULE #define UTIL_H_MODULE @@ -27,70 +24,63 @@ extern "C" { #endif -/* ************************************** -* Compiler Options -****************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ -#endif -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ - #define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ -#endif -#endif - - -/* Unix Large Files support (>4GB) */ -#if !defined(__LP64__) /* No point defining Large file for 64 bit */ -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ -# define _LARGEFILE_SOURCE /* fseeko, ftello */ -# elif !defined(_LARGEFILE64_SOURCE) -# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */ -# endif -#endif /*-**************************************** * Dependencies ******************************************/ -#include <stdlib.h> /* features.h with _POSIX_C_SOURCE, malloc */ -#include <stdio.h> /* fprintf */ -#include <string.h> /* strerr, strlen, memcpy */ -#include <stddef.h> /* ptrdiff_t */ -#include <sys/types.h> /* stat, utime */ -#include <sys/stat.h> /* stat */ +#include "platform.h" /* PLATFORM_POSIX_VERSION */ +#include <stdlib.h> /* malloc */ +#include <stddef.h> /* size_t, ptrdiff_t */ +#include <stdio.h> /* fprintf */ +#include <sys/types.h> /* stat, utime */ +#include <sys/stat.h> /* stat */ #if defined(_MSC_VER) - #include <sys/utime.h> /* utime */ - #include <io.h> /* _chmod */ +# include <sys/utime.h> /* utime */ +# include <io.h> /* _chmod */ #else - #include <unistd.h> /* chown, stat */ - #include <utime.h> /* utime */ +# include <unistd.h> /* chown, stat */ +# include <utime.h> /* utime */ #endif -#include <time.h> /* time */ +#include <time.h> /* time */ #include <errno.h> -/* ************************************* -* Constants -***************************************/ -#define LIST_SIZE_INCREASE (8*1024) +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; +#endif -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(__GNUC__) -# define UTIL_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define UTIL_STATIC static inline -#elif defined(_MSC_VER) -# define UTIL_STATIC static __inline + +/* ************************************************************ +* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW +***************************************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +# define UTIL_fseek _fseeki64 +#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define UTIL_fseek fseeko +#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) +# define UTIL_fseek fseeko64 #else -# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +# define UTIL_fseek fseek #endif @@ -99,88 +89,167 @@ extern "C" { ******************************************/ #if defined(_WIN32) # include <windows.h> -# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) +# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleepMilli(milli) Sleep(milli) -#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) +#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ # include <unistd.h> # include <sys/resource.h> /* setpriority */ # include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ # if defined(PRIO_PROCESS) -# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) +# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20) # else -# define SET_HIGH_PRIORITY /* disabled */ +# define SET_REALTIME_PRIORITY /* disabled */ # endif # define UTIL_sleep(s) sleep(s) -# if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L) +# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ # define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } # else # define UTIL_sleepMilli(milli) /* disabled */ # endif #else -# define SET_HIGH_PRIORITY /* disabled */ +# define SET_REALTIME_PRIORITY /* disabled */ # define UTIL_sleep(s) /* disabled */ # define UTIL_sleepMilli(milli) /* disabled */ #endif -/*-************************************************************** -* Basic Types -*****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; +/* ************************************* +* Constants +***************************************/ +#define LIST_SIZE_INCREASE (8*1024) + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(__INTEL_COMPILER) +# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */ +#endif +#if defined(__GNUC__) +# define UTIL_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define UTIL_STATIC static inline +#elif defined(_MSC_VER) +# define UTIL_STATIC static __inline #else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif +# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif /*-**************************************** * Time functions ******************************************/ -#if !defined(_WIN32) - typedef clock_t UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } -#else - typedef LARGE_INTEGER UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } +#if defined(_WIN32) /* Windows */ + typedef LARGE_INTEGER UTIL_time_t; + UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) + { + static LARGE_INTEGER ticksPerSecond; + static int init = 0; + if (!init) { + if (!QueryPerformanceFrequency(&ticksPerSecond)) + fprintf(stderr, "ERROR: QueryPerformanceFrequency() failure\n"); + init = 1; + } + return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; + } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) + { + static LARGE_INTEGER ticksPerSecond; + static int init = 0; + if (!init) { + if (!QueryPerformanceFrequency(&ticksPerSecond)) + fprintf(stderr, "ERROR: QueryPerformanceFrequency() failure\n"); + init = 1; + } + return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; + } +#elif defined(__APPLE__) && defined(__MACH__) + #include <mach/mach_time.h> + typedef U64 UTIL_time_t; + UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) + { + static mach_timebase_info_data_t rate; + static int init = 0; + if (!init) { + mach_timebase_info(&rate); + init = 1; + } + return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL; + } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) + { + static mach_timebase_info_data_t rate; + static int init = 0; + if (!init) { + mach_timebase_info(&rate); + init = 1; + } + return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom); + } +#elif (PLATFORM_POSIX_VERSION >= 200112L) + #include <time.h> + typedef struct timespec UTIL_time_t; + UTIL_STATIC UTIL_time_t UTIL_getTime(void) + { + UTIL_time_t now; + if (clock_gettime(CLOCK_MONOTONIC, &now)) + fprintf(stderr, "ERROR: Failed to get time\n"); /* we could also exit() */ + return now; + } + UTIL_STATIC UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end) + { + UTIL_time_t diff; + if (end.tv_nsec < begin.tv_nsec) { + diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec; + diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec; + } else { + diff.tv_sec = end.tv_sec - begin.tv_sec; + diff.tv_nsec = end.tv_nsec - begin.tv_nsec; + } + return diff; + } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end) + { + UTIL_time_t const diff = UTIL_getSpanTime(begin, end); + U64 micro = 0; + micro += 1000000ULL * diff.tv_sec; + micro += diff.tv_nsec / 1000ULL; + return micro; + } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end) + { + UTIL_time_t const diff = UTIL_getSpanTime(begin, end); + U64 nano = 0; + nano += 1000000000ULL * diff.tv_sec; + nano += diff.tv_nsec; + return nano; + } +#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */ + typedef clock_t UTIL_time_t; + UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } #endif /* returns time span in microseconds */ -UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond ) +UTIL_STATIC U64 UTIL_clockSpanMicro(UTIL_time_t clockStart) { - UTIL_time_t clockEnd; - UTIL_getTime(&clockEnd); - return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd); + UTIL_time_t const clockEnd = UTIL_getTime(); + return UTIL_getSpanTimeMicro(clockStart, clockEnd); } -UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) +UTIL_STATIC void UTIL_waitForNextTick(void) { - UTIL_time_t clockStart, clockEnd; - UTIL_getTime(&clockStart); + UTIL_time_t const clockStart = UTIL_getTime(); + UTIL_time_t clockEnd; do { - UTIL_getTime(&clockEnd); - } while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); + clockEnd = UTIL_getTime(); + } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0); } @@ -189,21 +258,27 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) * File functions ******************************************/ #if defined(_MSC_VER) - #define chmod _chmod - typedef struct _stat64 stat_t; + #define chmod _chmod + typedef struct __stat64 stat_t; #else typedef struct stat stat_t; #endif +UTIL_STATIC int UTIL_isRegFile(const char* infilename); + + UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) { int res = 0; struct utimbuf timebuf; - timebuf.actime = time(NULL); - timebuf.modtime = statbuf->st_mtime; - res += utime(filename, &timebuf); /* set access and modification times */ + if (!UTIL_isRegFile(filename)) + return -1; + + timebuf.actime = time(NULL); + timebuf.modtime = statbuf->st_mtime; + res += utime(filename, &timebuf); /* set access and modification times */ #if !defined(_WIN32) res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ @@ -230,63 +305,58 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) } -UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) +UTIL_STATIC int UTIL_isRegFile(const char* infilename) +{ + stat_t statbuf; + return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */ +} + + +UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) { int r; + stat_t statbuf; #if defined(_MSC_VER) - struct _stat64 statbuf; r = _stat64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; #else - struct stat statbuf; r = stat(infilename, &statbuf); - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ + if (!r && S_ISDIR(statbuf.st_mode)) return 1; #endif - return (U64)statbuf.st_size; -} - - -UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) -{ - U64 total = 0; - unsigned n; - for (n=0; n<nbFiles; n++) - total += UTIL_getFileSize(fileNamesTable[n]); - return total; + return 0; } -UTIL_STATIC int UTIL_doesFileExists(const char* infilename) +UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; #if defined(_MSC_VER) - struct _stat64 statbuf; + struct __stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#elif defined(__MINGW32__) && defined (__MSVCRT__) + struct _stati64 statbuf; + r = _stati64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ #else struct stat statbuf; r = stat(infilename, &statbuf); if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ #endif - return 1; + return (U64)statbuf.st_size; } -UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) +UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) { - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; -#else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (!r && S_ISDIR(statbuf.st_mode)) return 1; -#endif - return 0; + U64 total = 0; + unsigned n; + for (n=0; n<nbFiles; n++) + total += UTIL_getFileSize(fileNamesTable[n]); + return total; } + /* * A modified version of realloc(). * If UTIL_realloc() fails the original block is freed. @@ -307,7 +377,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ { char* path; int dirLength, fnameLength, pathLength, nbFiles = 0; - WIN32_FIND_DATA cFile; + WIN32_FIND_DATAA cFile; HANDLE hFile; dirLength = (int)strlen(dirName); @@ -319,7 +389,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ path[dirLength+1] = '*'; path[dirLength+2] = 0; - hFile=FindFirstFile(path, &cFile); + hFile=FindFirstFileA(path, &cFile); if (hFile == INVALID_HANDLE_VALUE) { fprintf(stderr, "Cannot open directory '%s'\n", dirName); return 0; @@ -356,16 +426,16 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ } } free(path); - } while (FindNextFile(hFile, &cFile)); + } while (FindNextFileA(hFile, &cFile)); FindClose(hFile); return nbFiles; } -#elif (defined(__APPLE__) && defined(__MACH__)) || \ - ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */ +#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # define UTIL_HAS_CREATEFILELIST # include <dirent.h> /* opendir, readdir */ +# include <string.h> /* strerror, memcpy */ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { @@ -427,7 +497,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { (void)bufStart; (void)bufEnd; (void)pos; - fprintf(stderr, "Directory %s ignored (lz4 compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); + fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); return 0; } @@ -443,16 +513,15 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i { size_t pos; unsigned i, nbFiles; - char *bufend, *buf; + char* buf = (char*)malloc(LIST_SIZE_INCREASE); + char* bufend = buf + LIST_SIZE_INCREASE; const char** fileTable; - buf = (char*)malloc(LIST_SIZE_INCREASE); if (!buf) return NULL; - bufend = buf + LIST_SIZE_INCREASE; for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) { if (!UTIL_isDirectory(inputNames[i])) { - size_t len = strlen(inputNames[i]); + size_t const len = strlen(inputNames[i]); if (buf + pos + len >= bufend) { ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; buf = (char*)UTIL_realloc(buf, newListSize); @@ -474,8 +543,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); if (!fileTable) { free(buf); return NULL; } - for (i=0, pos=0; i<nbFiles; i++) - { + for (i=0, pos=0; i<nbFiles; i++) { fileTable[i] = buf + pos; pos += strlen(fileTable[i]) + 1; } diff --git a/tests/Makefile b/tests/Makefile index e58d449..819ba43 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,6 +1,8 @@ # ########################################################################## # LZ4 programs - Makefile -# Copyright (C) Yann Collet 2011-2016 +# Copyright (C) Yann Collet 2011-2017 +# +# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets # # GPL v2 License # @@ -28,30 +30,28 @@ # datagen : generates synthetic data samples for tests & benchmarks # ########################################################################## -DESTDIR ?= -PREFIX ?= /usr/local -BINDIR := $(PREFIX)/bin -MANDIR := $(PREFIX)/share/man/man1 -LIBDIR := ../lib +LZ4DIR := ../lib PRGDIR := ../programs -VOID := /dev/null TESTDIR := versionsTest PYTHON ?= python3 -CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make -CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ - -Wdeclaration-after-statement -Wstrict-prototypes \ +DEBUGFLAGS = -g -DLZ4_DEBUG=1 +CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make +CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ + -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 -CFLAGS += $(MOREFLAGS) -CPPFLAGS:= -I$(LIBDIR) -DXXH_NAMESPACE=LZ4_ +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_ FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) -EXT =.exe +EXT =.exe +VOID = nul else -EXT = +EXT = +VOID = /dev/null endif LZ4 := $(PRGDIR)/lz4$(EXT) @@ -64,46 +64,47 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest datagen fasttest +all: fullbench fuzzer frametest datagen all32: CFLAGS+=-m32 all32: all lz4: - $(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)" + $(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)" -lz4c: - $(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)" +lz4c unlz4 lz4cat: lz4 + ln -sf $(LZ4) $(PRGDIR)/$@ lz4c32: # create a 32-bits version for 32/64 interop tests - $(MAKE) -C $(PRGDIR) clean $@ CFLAGS="-m32 $(CFLAGS)" - cp $(LZ4) $(LZ4)c32 + $(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)" -fullbench : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/lz4frame.o $(LIBDIR)/xxhash.o fullbench.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +%.o : $(LZ4DIR)/%.c $(LZ4DIR)/%.h + $(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@ -fullbench-lib: fullbench.c $(LIBDIR)/xxhash.c - $(MAKE) -C $(LIBDIR) liblz4.a - $(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/liblz4.a +fullbench : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c + $(CC) $(FLAGS) $^ -o $@$(EXT) -fullbench-dll: fullbench.c $(LIBDIR)/xxhash.c - $(MAKE) -C $(LIBDIR) liblz4 - $(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LIBDIR)/dll/liblz4.dll +$(LZ4DIR)/liblz4.a: + $(MAKE) -C $(LZ4DIR) liblz4.a -fuzzer : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/xxhash.o fuzzer.c +fullbench-lib: fullbench.c $(LZ4DIR)/liblz4.a $(CC) $(FLAGS) $^ -o $@$(EXT) -frametest: $(LIBDIR)/lz4frame.o $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/xxhash.o frametest.c +fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c + $(MAKE) -C $(LZ4DIR) liblz4 + $(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/liblz4.dll + +fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) -fasttest: $(LIBDIR)/lz4.o fasttest.c +frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c $(CC) $(FLAGS) $^ -o $@$(EXT) datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT) clean: - @$(MAKE) -C $(LIBDIR) $@ > $(VOID) + @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) @$(RM) core *.o *.test tmp* \ fullbench-dll$(EXT) fullbench-lib$(EXT) \ @@ -119,140 +120,216 @@ versionsTest: $(PYTHON) test-lz4-versions.py -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#----------------------------------------------------------------------------- +# validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) MD5:=md5sum ifneq (,$(filter $(shell uname), Darwin )) MD5:=md5 -r endif -test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer +DIFF:=diff +ifneq (,$(filter $(shell uname),SunOS)) +DIFF:=gdiff +endif + +DD:=dd + + +test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test32: CFLAGS+=-m32 test32: test test-lz4-sparse: lz4 datagen @echo "\n ---- test sparse file support ----" - ./datagen -g5M -P100 > tmpSrc - $(LZ4) -B4D tmpSrc | $(LZ4) -dv --sparse > tmpB4 - diff -s tmpSrc tmpB4 - $(LZ4) -B5D tmpSrc | $(LZ4) -dv --sparse > tmpB5 - diff -s tmpSrc tmpB5 - $(LZ4) -B6D tmpSrc | $(LZ4) -dv --sparse > tmpB6 - diff -s tmpSrc tmpB6 - $(LZ4) -B7D tmpSrc | $(LZ4) -dv --sparse > tmpB7 - diff -s tmpSrc tmpB7 - $(LZ4) tmpSrc | $(LZ4) -dv --no-sparse > tmpNoSparse - diff -s tmpSrc tmpNoSparse - ls -ls tmp* - ./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmpOdd # Odd size file (to generate non-full last block) - ./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd - ls -ls tmpOdd - @$(RM) tmp* + ./datagen -g5M -P100 > tmplsdg5M + $(LZ4) -B4D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB4 + $(DIFF) -s tmplsdg5M tmplscB4 + $(LZ4) -B5D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB5 + $(DIFF) -s tmplsdg5M tmplscB5 + $(LZ4) -B6D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB6 + $(DIFF) -s tmplsdg5M tmplscB6 + $(LZ4) -B7D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB7 + $(DIFF) -s tmplsdg5M tmplscB7 + $(LZ4) tmplsdg5M | $(LZ4) -dv --no-sparse > tmplsnosparse + $(DIFF) -s tmplsdg5M tmplsnosparse + ls -ls tmpls* + ./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block) + ./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd + ls -ls tmplsodd + @$(RM) tmpls* @echo "\n Compatibility with Console :" echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | cat echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c @echo "\n Compatibility with Append :" - ./datagen -P100 -g1M > tmp1M - cat tmp1M tmp1M > tmp2M - $(LZ4) -B5 -v tmp1M tmpC - $(LZ4) -d -v tmpC tmpR - $(LZ4) -d -v tmpC >> tmpR + ./datagen -P100 -g1M > tmplsdg1M + cat tmplsdg1M tmplsdg1M > tmpls2M + $(LZ4) -B5 -v tmplsdg1M tmplsc + $(LZ4) -d -v tmplsc tmplsr + $(LZ4) -d -v tmplsc >> tmplsr ls -ls tmp* - diff tmp2M tmpR - @$(RM) tmp* + $(DIFF) tmpls2M tmplsr + @$(RM) tmpls* test-lz4-contentSize: lz4 datagen @echo "\n ---- test original size support ----" - ./datagen -g15M > tmp - $(LZ4) -v tmp | $(LZ4) -t - $(LZ4) -v --content-size tmp | $(LZ4) -d > tmp2 - diff -s tmp tmp2 + ./datagen -g15M > tmplc1 + $(LZ4) -v tmplc1 | $(LZ4) -t + $(LZ4) -v --content-size tmplc1 | $(LZ4) -d > tmplc2 + $(DIFF) -s tmplc1 tmplc2 # test large size [2-4] GB - @./datagen -g3G -P100 | $(LZ4) --verbose | $(LZ4) --decompress --force --sparse - tmp - @ls -ls tmp - @./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmp2 - @ls -ls tmp2 - diff -s tmp tmp2 - @$(RM) tmp* + @./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmplc1 + @ls -ls tmplc1 + @./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmplc2 + @ls -ls tmplc2 + $(DIFF) -s tmplc1 tmplc2 + @$(RM) tmplc* test-lz4-frame-concatenation: lz4 datagen @echo "\n ---- test frame concatenation ----" - @echo -n > empty.test - @echo hi > nonempty.test - cat nonempty.test empty.test nonempty.test > orig.test - @$(LZ4) -zq empty.test > empty.lz4.test - @$(LZ4) -zq nonempty.test > nonempty.lz4.test - cat nonempty.lz4.test empty.lz4.test nonempty.lz4.test > concat.lz4.test - $(LZ4) -d concat.lz4.test > result.test - sdiff orig.test result.test - @$(RM) *.test + @echo -n > tmp-lfc-empty + @echo hi > tmp-lfc-nonempty + cat tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src + @$(LZ4) -zq tmp-lfc-empty > tmp-lfc-empty.lz4 + @$(LZ4) -zq tmp-lfc-nonempty > tmp-lfc-nonempty.lz4 + cat tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4 + $(LZ4) -d tmp-lfc-concat.lz4 > tmp-lfc-result + sdiff tmp-lfc-src tmp-lfc-result + @$(RM) tmp-lfc-* @echo frame concatenation test completed test-lz4-multiple: lz4 datagen @echo "\n ---- test multiple files ----" - @./datagen -s1 > tmp1 2> $(VOID) - @./datagen -s2 -g100K > tmp2 2> $(VOID) - @./datagen -s3 -g1M > tmp3 2> $(VOID) - $(LZ4) -f -m tmp* - ls -ls tmp* - @$(RM) tmp1 tmp2 tmp3 - $(LZ4) -df -m *.lz4 - ls -ls tmp* - $(LZ4) -f -m tmp1 notHere tmp2; echo $$? - @$(RM) tmp* - -test-lz4-basic: lz4 datagen + @./datagen -s1 > tmp-tlm1 2> $(VOID) + @./datagen -s2 -g100K > tmp-tlm2 2> $(VOID) + @./datagen -s3 -g1M > tmp-tlm3 2> $(VOID) + $(LZ4) -f -m tmp-tlm* + ls -ls tmp-tlm* + @$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3 + $(LZ4) -df -m tmp-tlm*.lz4 + ls -ls tmp-tlm* + $(LZ4) -f -m tmp-tlm1 notHere tmp-tlm2; echo $$? + @$(RM) tmp-tlm* + +test-lz4-basic: lz4 datagen unlz4 lz4cat @echo "\n ---- test lz4 basic compression/decompression ----" - ./datagen -g0 | $(LZ4) -v | $(LZ4) -t - ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t - ./datagen -g20KB > tmpSrc - $(LZ4) < tmpSrc | $(LZ4) -d > tmpRes - diff -q tmpSrc tmpRes - $(LZ4) --no-frame-crc < tmpSrc | $(LZ4) -d > tmpRes - diff -q tmpSrc tmpRes - ./datagen | $(LZ4) | $(LZ4) -t - ./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t - ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt - ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t - ./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t - @$(RM) tmp* + ./datagen -g0 | $(LZ4) -v | $(LZ4) -t + ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t + ./datagen -g20KB > tmp-tlb-dg20k + $(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec + $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec + $(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec + $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec + ./datagen | $(LZ4) | $(LZ4) -t + ./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t + ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt + ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t + ./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t + @echo "hello world" > tmp-tlb-hw + $(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4 + test ! -f tmp-tlb-hw # must fail (--rm) + test -f tmp-tlb-hw.lz4 + $(PRGDIR)/lz4cat tmp-tlb-hw.lz4 # must display hello world + test -f tmp-tlb-hw.lz4 + $(PRGDIR)/unlz4 --rm tmp-tlb-hw.lz4 tmp-tlb-hw + test -f tmp-tlb-hw + test ! -f tmp-tlb-hw.lz4 # must fail (--rm) + test ! -f tmp-tlb-hw.lz4.lz4 # must fail (unlz4) + $(PRGDIR)/lz4cat tmp-tlb-hw # pass-through mode + test -f tmp-tlb-hw + test ! -f tmp-tlb-hw.lz4 # must fail (lz4cat) + $(LZ4) tmp-tlb-hw tmp-tlb-hw.lz4 # creates tmp-tlb-hw.lz4 + $(PRGDIR)/lz4cat < tmp-tlb-hw.lz4 > tmp-tlb3 # checks lz4cat works with stdin (#285) + $(DIFF) -q tmp-tlb-hw tmp-tlb3 + $(PRGDIR)/lz4cat < tmp-tlb-hw > tmp-tlb2 # checks lz4cat works in pass-through mode + $(DIFF) -q tmp-tlb-hw tmp-tlb2 + cp tmp-tlb-hw ./-d + $(LZ4) --rm -- -d -d.lz4 # compresses ./d into ./-d.lz4 + test -f ./-d.lz4 + test ! -f ./-d + mv ./-d.lz4 ./-z + $(LZ4) -d --rm -- -z tmp-tlb4 # uncompresses ./-z into tmp-tlb4 + test ! -f ./-z + $(DIFF) -q tmp-tlb-hw tmp-tlb4 + $(LZ4) -f tmp-tlb-hw + cat tmp-tlb-hw >> tmp-tlb-hw.lz4 + $(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data + $(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum + @$(RM) tmp-tlb* + +test-lz4-dict: lz4 datagen + @echo "\n ---- test lz4 compression/decompression with dictionary ----" + ./datagen -g16KB > tmp-dict + ./datagen -g32KB > tmp-dict-sample-32k + < tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k + ./datagen -g128MB > tmp-dict-sample-128m + < tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m + touch tmp-dict-sample-0 + < tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0 + + < tmp-dict-sample-32k $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-32k + < tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0 + + @echo "\n ---- test lz4 dictionary loading ----" + ./datagen -g128KB > tmp-dict-data-128KB + set -e; \ + for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \ + ./datagen -g$$l > tmp-dict-$$l; \ + $(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \ + < tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \ + < tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \ + done + + @$(RM) tmp-dict* test-lz4-hugefile: lz4 datagen @echo "\n ---- test huge files compression/decompression ----" ./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt ./datagen -g6GB | $(LZ4) -v5BD | $(LZ4) -qt - @$(RM) tmp* test-lz4-testmode: lz4 datagen @echo "\n ---- bench mode ----" $(LZ4) -bi1 @echo "\n ---- test mode ----" - ./datagen | $(LZ4) -t && false || true - ./datagen | $(LZ4) -tf && false || true + ! ./datagen | $(LZ4) -t + ! ./datagen | $(LZ4) -tf @echo "\n ---- pass-through mode ----" - ./datagen | $(LZ4) -d > $(VOID) && false || true + ! ./datagen | $(LZ4) -d > $(VOID) ./datagen | $(LZ4) -df > $(VOID) - @echo "Hello World !" > tmp1 - $(LZ4) -dcf tmp1 - @echo "from underground..." > tmp2 - $(LZ4) -dcfm tmp1 tmp2 - @echo "\n ---- test cli ----" - $(LZ4) file-does-not-exist && false || true - $(LZ4) -f file-does-not-exist && false || true - $(LZ4) -fm file1-dne file2-dne && false || true - $(LZ4) -fm file1-dne file2-dne && false || true - -test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \ - test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \ - test-lz4-hugefile + @echo "Hello World !" > tmp-tlt1 + $(LZ4) -dcf tmp-tlt1 + @echo "from underground..." > tmp-tlt2 + $(LZ4) -dcfm tmp-tlt1 tmp-tlt2 + @echo "\n ---- non-existing source ----" + ! $(LZ4) file-does-not-exist + ! $(LZ4) -f file-does-not-exist + ! $(LZ4) -fm file1-dne file2-dne + @$(RM) tmp-tlt + +test-lz4-opt-parser: lz4 datagen + @echo "\n ---- test opt-parser ----" + ./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t + ./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t + ./datagen -g256K | $(LZ4) -12B4D | $(LZ4) -t + ./datagen -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t + ./datagen -g1M | $(LZ4) -12B5 | $(LZ4) -t + ./datagen -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t + ./datagen -g4M | $(LZ4) -11vq | $(LZ4) -qt + ./datagen -g8M | $(LZ4) -11B4 | $(LZ4) -t + ./datagen -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t + ./datagen -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t + +test-lz4: lz4 datagen test-lz4-basic test-lz4-opt-parser test-lz4-multiple \ + test-lz4-sparse test-lz4-frame-concatenation test-lz4-testmode \ + test-lz4-contentSize test-lz4-hugefile test-lz4-dict + @$(RM) tmp* test-lz4c: lz4c datagen - @echo "\n ---- test lz4c version ----" + @echo "\n ---- test lz4c variant ----" ./datagen -g256MB | $(LZ4)c -l -v | $(LZ4)c -t test-lz4c32: CFLAGS+=-m32 @@ -302,25 +379,22 @@ test-frametest: frametest test-frametest32: CFLAGS += -m32 test-frametest32: test-frametest -test-fasttest: fasttest - ./fasttest - test-mem: lz4 datagen fuzzer frametest fullbench @echo "\n ---- valgrind tests : memory analyzer ----" valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) - ./datagen -g16KB > tmp - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f tmp $(VOID) - ./datagen -g16KB -s2 > tmp2 - ./datagen -g16KB -s3 > tmp3 - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple tmp tmp2 tmp3 - ./datagen -g16MB > tmp - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f tmp tmp2 - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t tmp2 - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 tmp - valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2 - ./datagen -g256MB > tmp - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq tmp $(VOID) - $(RM) tmp* + ./datagen -g16KB > ftmdg16K + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID) + ./datagen -g16KB -s2 > ftmdg16K2 + ./datagen -g16KB -s3 > ftmdg16K3 + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3 + ./datagen -g16MB > ftmdg16M + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg16M ftmdg16K2 + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2 + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg16M + valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg16M ftmdg16K2 + ./datagen -g256MB > ftmdg256M + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg256M $(VOID) + $(RM) ftm* valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1 valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256 diff --git a/tests/datagencli.c b/tests/datagencli.c index 0b78d37..c985197 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -27,31 +27,13 @@ /************************************** * Includes **************************************/ +#include "util.h" /* U32 */ #include <stdio.h> /* fprintf, stderr */ #include "datagen.h" /* RDG_generate */ #include "lz4.h" /* LZ4_VERSION_STRING */ /************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** * Constants **************************************/ #define KB *(1 <<10) diff --git a/tests/fasttest.c b/tests/fasttest.c deleted file mode 100644 index a165293..0000000 --- a/tests/fasttest.c +++ /dev/null @@ -1,131 +0,0 @@ -/************************************** - * Compiler Options - **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS // for MSVC -# define snprintf sprintf_s -#endif -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif - - -/************************************** - * Includes - **************************************/ -#include <stdio.h> -#include <stdint.h> -#include <stdlib.h> -#include <string.h> -#include "lz4.h" - - -/* Returns non-zero on failure. */ -int test_compress(const char *input, int inSize, char *output, int outSize) -{ - LZ4_stream_t lz4Stream_body = { 0 }; - LZ4_stream_t* lz4Stream = &lz4Stream_body; - - int inOffset = 0; - int outOffset = 0; - - if (inSize & 3) return -1; - - while (inOffset < inSize) { - const int length = inSize >> 2; - if (inSize > 1024) return -2; - if (outSize - (outOffset + 8) < LZ4_compressBound(length)) return -3; - { - const int outBytes = LZ4_compress_fast_continue( - lz4Stream, input + inOffset, output + outOffset + 8, length, outSize-outOffset, 1); - if(outBytes <= 0) return -4; - memcpy(output + outOffset, &length, 4); /* input length */ - memcpy(output + outOffset + 4, &outBytes, 4); /* output length */ - inOffset += length; - outOffset += outBytes + 8; - } - } - if (outOffset + 8 > outSize) return -5; - memset(output + outOffset, 0, 4); - memset(output + outOffset + 4, 0, 4); - return 0; -} - -/* Returns non-zero on failure. Not a safe function. */ -int test_decompress(const char *uncompressed, const char *compressed) -{ - char outBufferA[1024]; - char spacing; /* So prefixEnd != dest */ - char outBufferB[1024]; - char *output = outBufferA; - char *lastOutput = outBufferB; - LZ4_streamDecode_t lz4StreamDecode_body = { 0 }; - LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; - int offset = 0; - int unOffset = 0; - int lastBytes = 0; - - (void)spacing; - - for(;;) { - int32_t bytes; - int32_t unBytes; - /* Read uncompressed size and compressed size */ - memcpy(&unBytes, compressed + offset, 4); - memcpy(&bytes, compressed + offset + 4, 4); - offset += 8; - /* Check if we reached end of stream or error */ - if(bytes == 0 && unBytes == 0) return 0; - if(bytes <= 0 || unBytes <= 0 || unBytes > 1024) return 1; - - /* Put the last output in the dictionary */ - LZ4_setStreamDecode(lz4StreamDecode, lastOutput, lastBytes); - /* Decompress */ - bytes = LZ4_decompress_fast_continue( - lz4StreamDecode, compressed + offset, output, unBytes); - if(bytes <= 0) return 2; - /* Check result */ - { int const r = memcmp(uncompressed + unOffset, output, unBytes); - if (r) return 3; - } - { char* const tmp = output; output = lastOutput; lastOutput = tmp; } - offset += bytes; - unOffset += unBytes; - lastBytes = unBytes; - } -} - - -int main(int argc, char **argv) -{ - char input[] = - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello"; - char output[LZ4_COMPRESSBOUND(4096)]; - int r; - - (void)argc; - (void)argv; - - if ((r = test_compress(input, sizeof(input), output, sizeof(output)))) { - return r; - } - if ((r = test_decompress(input, output))) { - return r; - } - return 0; -} diff --git a/tests/frametest.c b/tests/frametest.c index a99728f..88d0afd 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -35,35 +35,18 @@ /*-************************************ * Includes **************************************/ +#include "util.h" /* U32 */ #include <stdlib.h> /* malloc, free */ #include <stdio.h> /* fprintf */ #include <string.h> /* strcmp */ #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ +#include <assert.h> #include "lz4frame_static.h" #include "lz4.h" /* LZ4_VERSION_STRING */ #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" /* XXH64 */ -/*-************************************ -* Basic Types -**************************************/ -#if !defined(__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - - /* unoptimized version; solves endianess & alignment issues */ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) { @@ -85,7 +68,6 @@ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) #define GB *(1U<<30) static const U32 nbTestsDefault = 256 KB; -#define COMPRESSIBLE_NOISE_LENGTH (2 MB) #define FUZ_COMPRESSIBILITY_DEFAULT 50 static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; @@ -110,7 +92,7 @@ static clock_t g_clockTime = 0; *****************************************/ static U32 no_prompt = 0; static U32 displayLevel = 2; -static U32 pause = 0; +static U32 use_pause = 0; /*-******************************************************* @@ -182,9 +164,12 @@ static unsigned FUZ_highbit(U32 v32) /*-******************************************************* * Tests *********************************************************/ +#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) goto _output_error +#define CHECK(f) { LZ4F_errorCode_t const CHECK_V(err_ , f); } + int basicTests(U32 seed, double compressibility) { - int testResult = 0; +#define COMPRESSIBLE_NOISE_LENGTH (2 MB) void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL); void* const compressedBuffer = malloc(cBuffSize); @@ -194,9 +179,10 @@ int basicTests(U32 seed, double compressibility) LZ4F_decompressionContext_t dCtx = NULL; LZ4F_compressionContext_t cctx = NULL; U64 crcOrig; - + int basicTests_error = 0; LZ4F_preferences_t prefs; memset(&prefs, 0, sizeof(prefs)); + if (!CNBuffer || !compressedBuffer || !decodedBuffer) { DISPLAY("allocation error, not enough memory to start fuzzer tests \n"); goto _output_error; @@ -204,49 +190,56 @@ int basicTests(U32 seed, double compressibility) FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + /* LZ4F_compressBound() : special case : srcSize == 0 */ + DISPLAYLEVEL(3, "LZ4F_compressBound(0) = "); + { size_t const cBound = LZ4F_compressBound(0, NULL); + if (cBound < 64 KB) goto _output_error; + DISPLAYLEVEL(3, " %u \n", (U32)cBound); + } + /* Special case : null-content frame */ testSize = 0; - DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n"); - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize); + DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : "); + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL)); + DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize); DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n"); - { LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; } + CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n"); { size_t avail_in = cSize; LZ4F_frameInfo_t frame_info; - LZ4F_errorCode_t const errorCode = LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) ); } DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n"); - { LZ4F_errorCode_t const errorCode = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(errorCode)) goto _output_error; } + CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL; /* test one-pass frame compression */ testSize = COMPRESSIBLE_NOISE_LENGTH; - DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : \n"); - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); - if (LZ4F_isError(cSize)) goto _output_error; + + DISPLAYLEVEL(3, "LZ4F_compressFrame, using fast level -3 : "); + { LZ4F_preferences_t fastCompressPrefs; + memset(&fastCompressPrefs, 0, sizeof(fastCompressPrefs)); + fastCompressPrefs.compressionLevel = -3; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs)); + DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); + } + + DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : "); + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL)); DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); DISPLAYLEVEL(3, "Decompression test : \n"); { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; size_t compressedBufferSize = cSize; - BYTE* ip = (BYTE*)compressedBuffer; - BYTE* const iend = (BYTE*)compressedBuffer + cSize; - LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); - DISPLAYLEVEL(3, "Single Pass decompression : \n"); - { size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL); - if (LZ4F_isError(decompressError)) goto _output_error; } - { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + DISPLAYLEVEL(3, "Single Pass decompression : "); + CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) ); + { U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1); if (crcDest != crcOrig) goto _output_error; } DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize); @@ -259,8 +252,7 @@ int basicTests(U32 seed, double compressibility) BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH; DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes); - decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL); - if (LZ4F_isError(decResult)) goto _output_error; + CHECK_V(decResult, LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL)); if (decResult != missingBytes) { DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult); goto _output_error; @@ -281,72 +273,93 @@ int basicTests(U32 seed, double compressibility) { size_t oSize = 0; size_t iSize = 0; LZ4F_frameInfo_t fi; + const BYTE* ip = (BYTE*)compressedBuffer; DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); - errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); - - DISPLAYLEVEL(3, "get FrameInfo on null input : "); - errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); - if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; - DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); - - DISPLAYLEVEL(3, "get FrameInfo on not enough input : "); - iSize = 6; - errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); - if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; - DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); - ip += iSize; + CHECK( LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL) ); + //DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); + DISPLAYLEVEL(3, " OK \n"); + + DISPLAYLEVEL(3, "LZ4F_getFrameInfo on zero-size input : "); + { size_t nullSize = 0; + size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize); + if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) { + DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", + LZ4F_getErrorName(fiError)); + goto _output_error; + } + DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError)); + } - DISPLAYLEVEL(3, "get FrameInfo on enough input : "); + DISPLAYLEVEL(3, "LZ4F_getFrameInfo on not enough input : "); + { size_t inputSize = 6; + size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &inputSize); + if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) { + DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError)); + goto _output_error; + } + DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError)); + } + + DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : "); iSize = 15 - iSize; - errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) ); DISPLAYLEVEL(3, " correctly decoded \n"); ip += iSize; } - DISPLAYLEVEL(3, "Byte after byte : \n"); + DISPLAYLEVEL(3, "Decode a buggy input : "); + assert(COMPRESSIBLE_NOISE_LENGTH > 64); + assert(cSize > 48); + memcpy(decodedBuffer, (char*)compressedBuffer+16, 32); /* save correct data */ + memcpy((char*)compressedBuffer+16, (const char*)decodedBuffer+32, 32); /* insert noise */ + { size_t dbSize = COMPRESSIBLE_NOISE_LENGTH; + size_t cbSize = cSize; + size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &dbSize, + compressedBuffer, &cbSize, + NULL); + if (!LZ4F_isError(decompressError)) goto _output_error; + DISPLAYLEVEL(3, "error detected : %s \n", LZ4F_getErrorName(decompressError)); + } + memcpy((char*)compressedBuffer+16, decodedBuffer, 32); /* restore correct data */ + + DISPLAYLEVEL(3, "Reset decompression context, since it's left in error state \n"); + LZ4F_resetDecompressionContext(dCtx); /* always successful */ + + DISPLAYLEVEL(3, "Byte after byte : "); { BYTE* const ostart = (BYTE*)decodedBuffer; BYTE* op = ostart; BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; + const BYTE* ip = (const BYTE*) compressedBuffer; + const BYTE* const iend = ip + cSize; while (ip < iend) { size_t oSize = oend-op; size_t iSize = 1; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; } { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); if (crcDest != crcOrig) goto _output_error; } DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH); - } - - errorCode = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(errorCode)) goto _output_error; - dCtx = NULL; + } } - DISPLAYLEVEL(3, "Using 64 KB block : \n"); + DISPLAYLEVEL(3, "Using 64 KB block : "); prefs.frameInfo.blockSizeID = LZ4F_max64KB; prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAYLEVEL(3, "without checksum : \n"); + DISPLAYLEVEL(3, "without checksum : "); prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAYLEVEL(3, "Using 256 KB block : \n"); + DISPLAYLEVEL(3, "Using 256 KB block : "); prefs.frameInfo.blockSizeID = LZ4F_max256KB; prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Decompression test : \n"); @@ -358,17 +371,13 @@ int basicTests(U32 seed, double compressibility) const BYTE* ip = (const BYTE*)compressedBuffer; const BYTE* const iend = (const BYTE*)compressedBuffer + cSize; - { LZ4F_errorCode_t const createError = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(createError)) goto _output_error; } - - DISPLAYLEVEL(3, "random segment sizes : \n"); + DISPLAYLEVEL(3, "random segment sizes : "); while (ip < iend) { unsigned const nbBits = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; size_t oSize = oend-op; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - { size_t const decompressError = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(decompressError)) goto _output_error; } + CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; } @@ -378,99 +387,269 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); } - { LZ4F_errorCode_t const freeError = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(freeError)) goto _output_error; } + CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL; } - DISPLAYLEVEL(3, "without checksum : \n"); + DISPLAYLEVEL(3, "without checksum : "); prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAYLEVEL(3, "Using 1 MB block : \n"); + DISPLAYLEVEL(3, "Using 1 MB block : "); prefs.frameInfo.blockSizeID = LZ4F_max1MB; prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAYLEVEL(3, "without checksum : \n"); + DISPLAYLEVEL(3, "without frame checksum : "); prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAYLEVEL(3, "Using 4 MB block : \n"); + DISPLAYLEVEL(3, "Using 4 MB block : "); prefs.frameInfo.blockSizeID = LZ4F_max4MB; prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); - DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity) - cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity) + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) ); DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); } - DISPLAYLEVEL(3, "without checksum : \n"); + DISPLAYLEVEL(3, "without frame checksum : "); prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); - DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity) - cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity) + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) ); DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); } - { size_t errorCode; + DISPLAYLEVEL(3, "LZ4F_compressFrame with block checksum : "); + memset(&prefs, 0, sizeof(prefs)); + prefs.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + + DISPLAYLEVEL(3, "Decompress with block checksum : "); + { size_t iSize = cSize; + size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; + LZ4F_decompressionContext_t dctx; + CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); + CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) ); + if (decodedSize != testSize) goto _output_error; + if (iSize != cSize) goto _output_error; + { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1); + U64 const crcSrc = XXH64(CNBuffer, testSize, 1); + if (crcDest != crcSrc) goto _output_error; + } + DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); + + CHECK( LZ4F_freeDecompressionContext(dctx) ); + } + + /* frame content size tests */ + { size_t cErr; BYTE* const ostart = (BYTE*)compressedBuffer; BYTE* op = ostart; - errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); - DISPLAYLEVEL(3, "compress without frameSize : \n"); + DISPLAYLEVEL(3, "compress without frameSize : "); memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo)); - errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); + op += cErr; + CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); + op += cErr; + CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) ); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); - DISPLAYLEVEL(3, "compress with frameSize : \n"); + DISPLAYLEVEL(3, "compress with frameSize : "); prefs.frameInfo.contentSize = testSize; op = ostart; - errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); + op += cErr; + CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); + op += cErr; + CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) ); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); - DISPLAYLEVEL(3, "compress with wrong frameSize : \n"); + DISPLAYLEVEL(3, "compress with wrong frameSize : "); prefs.frameInfo.contentSize = testSize+1; op = ostart; - errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL); - if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); } - else - goto _output_error; - - errorCode = LZ4F_freeCompressionContext(cctx); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); + op += cErr; + CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); + op += cErr; + cErr = LZ4F_compressEnd(cctx, op, testSize, NULL); + if (!LZ4F_isError(cErr)) goto _output_error; + DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(cErr)); + + CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL; } + /* dictID tests */ + { size_t cErr; + U32 const dictID = 0x99; + CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); + + DISPLAYLEVEL(3, "insert a dictID : "); + memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo)); + prefs.frameInfo.dictID = dictID; + CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); + DISPLAYLEVEL(3, "created frame header of size %i bytes \n", (int)cErr); + + DISPLAYLEVEL(3, "read a dictID : "); + CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); + memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo)); + CHECK( LZ4F_getFrameInfo(dCtx, &prefs.frameInfo, compressedBuffer, &cErr) ); + if (prefs.frameInfo.dictID != dictID) goto _output_error; + DISPLAYLEVEL(3, "%u \n", (U32)prefs.frameInfo.dictID); + + CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL; + CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL; + } + + + /* Dictionary compression test */ + { size_t const dictSize = 63 KB; + size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL); + size_t cSizeNoDict, cSizeWithDict; + LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize); + if (cdict == NULL) goto _output_error; + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : "); + CHECK_V(cSizeNoDict, + LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, + CNBuffer, dictSize, + NULL, NULL) ); + DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict); + + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : "); + CHECK_V(cSizeWithDict, + LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, + CNBuffer, dictSize, + cdict, NULL) ); + DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", + (unsigned)dictSize, (unsigned)cSizeWithDict); + if (cSizeWithDict >= cSizeNoDict) goto _output_error; /* must be more efficient */ + crcOrig = XXH64(CNBuffer, dictSize, 0); + + DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : "); + { LZ4F_dctx* dctx; + size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; + size_t compressedSize = cSizeWithDict; + CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); + CHECK( LZ4F_decompress_usingDict(dctx, + decodedBuffer, &decodedSize, + compressedBuffer, &compressedSize, + CNBuffer, dictSize, + NULL) ); + if (compressedSize != cSizeWithDict) goto _output_error; + if (decodedSize != dictSize) goto _output_error; + { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); + if (crcDest != crcOrig) goto _output_error; } + DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); + CHECK( LZ4F_freeDecompressionContext(dctx) ); + } + + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, negative level : "); + { size_t cSizeLevelMax; + LZ4F_preferences_t cParams; + memset(&cParams, 0, sizeof(cParams)); + cParams.compressionLevel = -3; + CHECK_V(cSizeLevelMax, + LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, + CNBuffer, dictSize, + cdict, &cParams) ); + DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax); + } + + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, level max : "); + { size_t cSizeLevelMax; + LZ4F_preferences_t cParams; + memset(&cParams, 0, sizeof(cParams)); + cParams.compressionLevel = LZ4F_compressionLevel_max(); + CHECK_V(cSizeLevelMax, + LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, + CNBuffer, dictSize, + cdict, &cParams) ); + DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax); + } + + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple linked blocks : "); + { size_t cSizeContiguous; + size_t const inSize = dictSize * 3; + size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL); + LZ4F_preferences_t cParams; + memset(&cParams, 0, sizeof(cParams)); + cParams.frameInfo.blockMode = LZ4F_blockLinked; + cParams.frameInfo.blockSizeID = LZ4F_max64KB; + CHECK_V(cSizeContiguous, + LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity, + CNBuffer, inSize, + cdict, &cParams) ); + DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", + (unsigned)inSize, (unsigned)cSizeContiguous); + + DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple linked blocks : "); + { LZ4F_dctx* dctx; + size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; + size_t compressedSize = cSizeContiguous; + CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); + CHECK( LZ4F_decompress_usingDict(dctx, + decodedBuffer, &decodedSize, + compressedBuffer, &compressedSize, + CNBuffer, dictSize, + NULL) ); + if (compressedSize != cSizeContiguous) goto _output_error; + if (decodedSize != inSize) goto _output_error; + crcOrig = XXH64(CNBuffer, inSize, 0); + { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); + if (crcDest != crcOrig) goto _output_error; } + DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); + CHECK( LZ4F_freeDecompressionContext(dctx) ); + } + } + + + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple independent blocks : "); + { size_t cSizeIndep; + size_t const inSize = dictSize * 3; + size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL); + LZ4F_preferences_t cParams; + memset(&cParams, 0, sizeof(cParams)); + cParams.frameInfo.blockMode = LZ4F_blockIndependent; + cParams.frameInfo.blockSizeID = LZ4F_max64KB; + CHECK_V(cSizeIndep, + LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity, + CNBuffer, inSize, + cdict, &cParams) ); + DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", + (unsigned)inSize, (unsigned)cSizeIndep); + + DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple independent blocks : "); + { LZ4F_dctx* dctx; + size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; + size_t compressedSize = cSizeIndep; + CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); + CHECK( LZ4F_decompress_usingDict(dctx, + decodedBuffer, &decodedSize, + compressedBuffer, &compressedSize, + CNBuffer, dictSize, + NULL) ); + if (compressedSize != cSizeIndep) goto _output_error; + if (decodedSize != inSize) goto _output_error; + crcOrig = XXH64(CNBuffer, inSize, 0); + { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); + if (crcDest != crcOrig) goto _output_error; } + DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); + CHECK( LZ4F_freeDecompressionContext(dctx) ); + } + } + + LZ4F_freeCDict(cdict); + } + + DISPLAYLEVEL(3, "Skippable frame test : \n"); { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); @@ -479,8 +658,7 @@ int basicTests(U32 seed, double compressibility) BYTE* ip = (BYTE*)compressedBuffer; BYTE* iend = (BYTE*)compressedBuffer + cSize + 8; - LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); /* generate skippable frame */ FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START); @@ -492,8 +670,7 @@ int basicTests(U32 seed, double compressibility) size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; size_t oSize = oend-op; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; } @@ -508,12 +685,11 @@ int basicTests(U32 seed, double compressibility) iend = ip+8; while (ip < iend) { - unsigned nbBits = FUZ_rand(&randState) % maxBits; + unsigned const nbBits = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; size_t oSize = oend-op; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; } @@ -529,8 +705,7 @@ int basicTests(U32 seed, double compressibility) size_t iSize = 10; size_t oSize = 10; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; } @@ -544,10 +719,10 @@ _end: free(decodedBuffer); LZ4F_freeDecompressionContext(dCtx); dCtx = NULL; LZ4F_freeCompressionContext(cctx); cctx = NULL; - return testResult; + return basicTests_error; _output_error: - testResult = 1; + basicTests_error = 1; DISPLAY("Error detected ! \n"); goto _end; } @@ -582,6 +757,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi size_t result; clock_t const startClock = clock(); clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; +# undef CHECK # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } @@ -618,10 +794,11 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi memset(&prefs, 0, sizeof(prefs)); prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1); prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3)); + prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)(FUZ_rand(&randState) & 1); prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1); prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0; prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2; - prefs.compressionLevel = FUZ_rand(&randState) % 5; + prefs.compressionLevel = -5 + (int)(FUZ_rand(&randState) % 11); if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL; DISPLAYUPDATE(2, "\r%5u ", testNb); @@ -723,7 +900,7 @@ _end: free(compressedBuffer); free(decodedBuffer); - if (pause) { + if (use_pause) { DISPLAY("press enter to finish \n"); (void)getchar(); } @@ -795,7 +972,7 @@ int main(int argc, const char** argv) break; case 'p': /* pause at the end */ argument++; - pause = 1; + use_pause = 1; break; case 'i': diff --git a/tests/fullbench.c b/tests/fullbench.c index 7d54492..f489392 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -23,20 +23,6 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -/************************************** -* Compiler Options -**************************************/ -/* Disable some Visual warning messages */ -#define _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ - -/* Unix Large Files support (>4GB) */ -#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions -# define _LARGEFILE_SOURCE -# define _FILE_OFFSET_BITS 64 -#elif ! defined(__LP64__) // No point defining Large file for 64 bit -# define _LARGEFILE64_SOURCE -#endif // S_ISREG & gettimeofday() are not supported by MSVC #if defined(_MSC_VER) || defined(_WIN32) @@ -47,8 +33,10 @@ /************************************** * Includes **************************************/ +#include "platform.h" /* _CRT_SECURE_NO_WARNINGS, Large Files support */ +#include "util.h" /* U32, UTIL_getFileSize */ #include <stdlib.h> /* malloc, free */ -#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <stdio.h> /* fprintf, fopen, ftello */ #include <sys/types.h> /* stat64 */ #include <sys/stat.h> /* stat64 */ #include <string.h> /* strcmp */ @@ -62,34 +50,6 @@ /************************************** -* Compiler Options -**************************************/ -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - - -/************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** * Constants **************************************/ #define PROGRAM_DESCRIPTION "LZ4 speed analyzer" @@ -194,21 +154,6 @@ static size_t BMK_findMaxMem(U64 requiredMem) } -static U64 BMK_GetFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - - /********************************************************* * Benchmark function *********************************************************/ @@ -398,7 +343,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } /* Memory size adjustments */ - inFileSize = BMK_GetFileSize(inFileName); + inFileSize = UTIL_getFileSize(inFileName); if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; } benchedSize = BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */ if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 06cfd40..c134fe3 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1,6 +1,6 @@ /* fuzzer.c - Fuzzer test tool for LZ4 - Copyright (C) Yann Collet 2012-2016 + Copyright (C) Yann Collet 2012-2017 GPL v2 License @@ -27,20 +27,23 @@ * Compiler options **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ # pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ #endif +#define LZ4_DISABLE_DEPRECATE_WARNINGS /*-************************************ * Dependencies **************************************/ +#include "platform.h" /* _CRT_SECURE_NO_WARNINGS */ +#include "util.h" /* U32 */ #include <stdlib.h> #include <stdio.h> /* fgets, sscanf */ #include <string.h> /* strcmp */ #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ +#define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" @@ -49,19 +52,7 @@ /*-************************************ * Basic Types **************************************/ -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include <stdint.h> -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; +#if !defined(__cplusplus) && !(defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which doesn't need address overflow test) */ #endif @@ -314,6 +305,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; int const dictSize = MIN(dictSizeRand, blockStart); + int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1); char* const block = ((char*)CNBuffer) + blockStart; const char* dict = block - dictSize; int compressedSize, HCcompressedSize; @@ -353,31 +345,67 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } DISPLAYLEVEL(5, " OK \n"); - } - else + } else { DISPLAYLEVEL(5, " \n"); - } + } } + + /* Test compression HC destSize */ + FUZ_DISPLAYTEST; + { int srcSize = blockSize; + int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; + char const endCheck = FUZ_rand(&randState) & 255; + void* ctx = LZ4_createHC(block); + FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed"); + compressedBuffer[targetSize] = endCheck; + ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel); + DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ", + compressionLevel, ret, targetSize, srcSize, blockSize); + LZ4_freeHC(ctx); + FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !"); + FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !"); + FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !"); + DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ", + compressionLevel, ret, targetSize, srcSize, blockSize); + if (targetSize>0) { + /* check correctness */ + U32 const crcBase = XXH32(block, srcSize, 0); + char const canary = FUZ_rand(&randState) & 255; + FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed"); + FUZ_DISPLAYTEST; + compressedSize = ret; + decodedBuffer[srcSize] = canary; + ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize"); + FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); + { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); + FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); + } + DISPLAYLEVEL(5, " OK \n"); + } else { + DISPLAYLEVEL(5, " \n"); + } } /* Test compression HC */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); + FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed"); HCcompressedSize = ret; /* Test compression HC using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); + FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed"); /* Test compression using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); - FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); + ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8); + FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed"); /* Test compression */ FUZ_DISPLAYTEST; ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress() failed"); + FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed"); compressedSize = ret; /* Decompression tests */ @@ -433,8 +461,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c // Test decoding with output size being 10 bytes too short => must fail FUZ_DISPLAYTEST; - if (blockSize>10) - { + if (blockSize>10) { decodedBuffer[blockSize-10] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short"); @@ -468,22 +495,22 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); + FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed despite sufficient space"); /* Test compression with output size being exactly what's necessary and external state (should work) */ FUZ_DISPLAYTEST; ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1); - FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space"); + FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, 9); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); + FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, 9); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); + FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed despite sufficient space"); /* Test compression with missing bytes into output buffer => must fail */ FUZ_DISPLAYTEST; @@ -492,8 +519,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[compressedSize-missingBytes] = 0; ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes); - FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); - FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) + FUZ_CHECKTEST(ret, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes); + FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_default overran output buffer ! (%i missingBytes)", missingBytes) } /* Test HC compression with missing bytes into output buffer => must fail */ @@ -502,9 +529,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, 9); - FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); - FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel); + FUZ_CHECKTEST(ret, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes); + FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compress_HC overran output buffer ! (%i missingBytes)", missingBytes) } @@ -518,14 +545,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_resetStream(&LZ4_stream); LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1); /* Just to fill hash tables */ blockContinueCompressedSize = LZ4_compress_fast_continue (&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed"); } /* Decompress with dictionary as prefix */ FUZ_DISPLAYTEST; memcpy(decodedBuffer, dict, dictSize); ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input"); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); if (crcCheck!=crcOrig) { int i=0; @@ -533,7 +560,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c printf("Wrong Byte at position %i/%i\n", i, blockSize); } - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); FUZ_DISPLAYTEST; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); @@ -547,18 +574,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_loadDict(&LZ4dict, dict, dictSize); blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed"); FUZ_DISPLAYTEST; LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1); - FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); FUZ_DISPLAYTEST; LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); + FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer"); /* Decompress with dictionary as external */ FUZ_DISPLAYTEST; @@ -603,32 +630,59 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST; dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7); + LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); + LZ4_setCompressionLevel(&LZ4dictHC, compressionLevel-1); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed"); FUZ_DISPLAYTEST; LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); + FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); FUZ_DISPLAYTEST; LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer"); FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); + crcCheck = XXH32(decodedBuffer, blockSize, 0); if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + /* Compress HC continue destSize */ + FUZ_DISPLAYTEST; + { int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5; + int consumedSize = blockSize; + FUZ_DISPLAYTEST; + LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); + LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); + blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace); + DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed"); + FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow"); + FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow"); + + FUZ_DISPLAYTEST; + decodedBuffer[consumedSize] = 0; + ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize); + FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); + FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") + { U32 const crcSrc = XXH32(block, consumedSize, 0); + U32 const crcDst = XXH32(decodedBuffer, consumedSize, 0); + if (crcSrc!=crcDst) + FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data"); + } + } + /* ***** End of tests *** */ /* Fill stats */ bytes += blockSize; @@ -666,7 +720,7 @@ _output_error: #define testCompressedSize (128 KB) #define ringBufferSize (8 KB) -static void FUZ_unitTests(void) +static void FUZ_unitTests(int compressionLevel) { const unsigned testNb = 0; const unsigned seed = 0; @@ -698,7 +752,7 @@ static void FUZ_unitTests(void) crcOrig = XXH64(testInput, testCompressedSize, 0); LZ4_resetStream(&streamingState); result = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1); - FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed"); + FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed!"); result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); @@ -728,7 +782,7 @@ static void FUZ_unitTests(void) memcpy (ringBuffer + rNext, testInput + iNext, messageSize); result = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1); - FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed"); + FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed"); result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed"); @@ -761,7 +815,7 @@ static void FUZ_unitTests(void) /* simple HC compression test */ crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); @@ -772,7 +826,7 @@ static void FUZ_unitTests(void) /* simple dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 64 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); @@ -786,7 +840,7 @@ static void FUZ_unitTests(void) { int result1, result2; int segSize = testCompressedSize / 2; crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); @@ -803,7 +857,7 @@ static void FUZ_unitTests(void) /* remote dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 32 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); @@ -824,7 +878,7 @@ static void FUZ_unitTests(void) int segSize = (FUZ_rand(&randState) & 8191); int segNb = 1; - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, dict, dictSize); XXH64_reset(&crcOrigState, 0); @@ -845,7 +899,6 @@ static void FUZ_unitTests(void) } dict = dst; - //dict = testInput + segStart; dictSize = segSize; dst += segSize + 1; @@ -870,7 +923,7 @@ static void FUZ_unitTests(void) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNew, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeState, NULL, 0); while (iNext + messageSize < testCompressedSize) { @@ -912,35 +965,34 @@ static void FUZ_unitTests(void) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNew, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeState, NULL, 0); #define BSIZE1 65537 #define BSIZE2 16435 /* first block */ + messageSize = BSIZE1; + XXH64_update(&xxhOrig, testInput + iNext, messageSize); + crcOrig = XXH64_digest(&xxhOrig); - messageSize = BSIZE1; - XXH64_update(&xxhOrig, testInput + iNext, messageSize); - crcOrig = XXH64_digest(&xxhOrig); - - result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); + result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); + result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); + FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); - XXH64_update(&xxhNew, testVerify + dNext, messageSize); - { U64 const crcNew = XXH64_digest(&xxhNew); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } + XXH64_update(&xxhNew, testVerify + dNext, messageSize); + { U64 const crcNew = XXH64_digest(&xxhNew); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } - /* prepare next message */ - dNext += messageSize; - totalMessageSize += messageSize; - messageSize = BSIZE2; - iNext = 132000; - memcpy(testInput + iNext, testInput + 8, messageSize); - if (dNext > dBufferSize) dNext = 0; + /* prepare next message */ + dNext += messageSize; + totalMessageSize += messageSize; + messageSize = BSIZE2; + iNext = 132000; + memcpy(testInput + iNext, testInput + 8, messageSize); + if (dNext > dBufferSize) dNext = 0; while (totalMessageSize < 9 MB) { XXH64_update(&xxhOrig, testInput + iNext, messageSize); @@ -967,7 +1019,7 @@ static void FUZ_unitTests(void) } } - printf("All unit tests completed successfully \n"); + printf("All unit tests completed successfully compressionLevel=%d \n", compressionLevel); return; _output_error: exit(1); @@ -1000,7 +1052,7 @@ int main(int argc, const char** argv) int nbTests = NB_ATTEMPTS; int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; - int pause = 0; + int use_pause = 0; const char* programName = argv[0]; U32 duration = 0; @@ -1012,7 +1064,7 @@ int main(int argc, const char** argv) // Decode command (note : aggregated commands are allowed) if (argument[0]=='-') { - if (!strcmp(argument, "--no-prompt")) { pause=0; seedset=1; g_displayLevel=1; continue; } + if (!strcmp(argument, "--no-prompt")) { use_pause=0; seedset=1; g_displayLevel=1; continue; } argument++; while (*argument!=0) { @@ -1028,7 +1080,7 @@ int main(int argc, const char** argv) case 'p': /* pause at the end */ argument++; - pause=1; + use_pause=1; break; case 'i': @@ -1102,7 +1154,7 @@ int main(int argc, const char** argv) } } - printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); + printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_versionString()); if (!seedset) { time_t const t = time(NULL); @@ -1113,12 +1165,12 @@ int main(int argc, const char** argv) if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - if ((seedset==0) && (testNb==0)) FUZ_unitTests(); + if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); } if (nbTests<=0) nbTests=1; { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); - if (pause) { + if (use_pause) { DISPLAY("press enter ... \n"); (void)getchar(); } diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj index aaf81ad..e24f961 100644 --- a/visual/VS2010/datagen/datagen.vcxproj +++ b/visual/VS2010/datagen/datagen.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -107,6 +108,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -124,6 +126,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -144,6 +147,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj index 363c5ae..3196768 100644 --- a/visual/VS2010/frametest/frametest.vcxproj +++ b/visual/VS2010/frametest/frametest.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -66,20 +66,20 @@ <PropertyGroup Label="UserMacros" /> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -107,6 +108,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -124,6 +126,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -144,6 +147,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj index 16a4f0d..8f503f5 100644 --- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj +++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -66,20 +66,20 @@ <PropertyGroup Label="UserMacros" /> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -109,6 +110,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -128,6 +130,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -150,6 +153,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj index 8d0a623..aa67431 100644 --- a/visual/VS2010/fullbench/fullbench.vcxproj +++ b/visual/VS2010/fullbench/fullbench.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -66,20 +66,20 @@ <PropertyGroup Label="UserMacros" /> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -107,6 +108,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -124,6 +126,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -144,6 +147,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj index 6672aaa..21cbf56 100644 --- a/visual/VS2010/fuzzer/fuzzer.vcxproj +++ b/visual/VS2010/fuzzer/fuzzer.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -66,20 +66,20 @@ <PropertyGroup Label="UserMacros" /> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -107,6 +108,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -124,6 +126,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -144,6 +147,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj index 389f13c..56ec3b9 100644 --- a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj +++ b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj @@ -40,14 +40,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>DynamicLibrary</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>DynamicLibrary</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -96,6 +96,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -111,6 +112,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -127,6 +129,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -146,6 +149,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> diff --git a/visual/VS2010/liblz4/liblz4.vcxproj b/visual/VS2010/liblz4/liblz4.vcxproj index a0b8000..61ea159 100644 --- a/visual/VS2010/liblz4/liblz4.vcxproj +++ b/visual/VS2010/liblz4/liblz4.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>StaticLibrary</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>StaticLibrary</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -95,6 +95,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -110,6 +111,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -126,6 +128,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -145,6 +148,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj index 295a94b..de7a714 100644 --- a/visual/VS2010/lz4/lz4.vcxproj +++ b/visual/VS2010/lz4/lz4.vcxproj @@ -29,24 +29,24 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>true</UseDebugLibraries> - <CharacterSet>MultiByte</CharacterSet> + <CharacterSet>Unicode</CharacterSet> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>true</UseDebugLibraries> - <CharacterSet>MultiByte</CharacterSet> + <CharacterSet>Unicode</CharacterSet> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> - <CharacterSet>MultiByte</CharacterSet> + <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> - <CharacterSet>MultiByte</CharacterSet> + <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -108,6 +109,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -126,6 +128,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -147,6 +150,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> |