diff options
35 files changed, 1493 insertions, 730 deletions
diff --git a/.travis.yml b/.travis.yml index e3f476b..dc61505 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,9 @@ matrix: include: # OS X Mavericks - os: osx - env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang + install: + - export CC=clang + env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lz4 MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" && CFLAGS=-m32 make -C tests clean test-lz4-contentSize' COMPILER=clang # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - os: linux @@ -52,7 +52,10 @@ endif .PHONY: default all lib lz4 clean test versionsTest examples -default: lib lz4-release +default: + @$(MAKE) -C $(LZ4DIR) + @$(MAKE) -C $(PRGDIR) + @cp $(PRGDIR)/lz4$(EXT) . all: @$(MAKE) -C $(LZ4DIR) $@ @@ -96,10 +99,10 @@ uninstall: @$(MAKE) -C $(PRGDIR) $@ travis-install: - $(MAKE) install PREFIX=~/install_test_dir + $(MAKE) -j1 install PREFIX=~/install_test_dir test: - $(MAKE) -C $(TESTDIR) test + $(MAKE) -C $(TESTDIR) $@ clangtest: clean clang -v @@ -1,3 +1,12 @@ +v1.7.5 +lz4hc : new high compression mode : levels 10-12 compress more and slower, by Przemyslaw Skibinski +lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) +cli : fix minor notification when using -r recursive mode +API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) +doc : markdown version of man page, by Takayuki Matsuoka (#279) +build : Makefile : fix make -jX lib+exe concurrency (#277) +build : cmake : improvements by Michał Górny (#296) + v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) diff --git a/contrib/cmake_unofficial/.gitignore b/contrib/cmake_unofficial/.gitignore index 0f81929..d39505d 100644 --- a/contrib/cmake_unofficial/.gitignore +++ b/contrib/cmake_unofficial/.gitignore @@ -5,3 +5,5 @@ CMakeFiles *.cmake Makefile liblz4.pc +lz4c +install_manifest.txt diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index de070d6..9a0983d 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -65,6 +65,11 @@ endif(NOT LZ4_BUNDLED_MODE AND NOT CPack_CMake_INCLUDED) # which case we always use static libraries. include(CMakeDependentOption) CMAKE_DEPENDENT_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON "NOT LZ4_BUNDLED_MODE" OFF) +CMAKE_DEPENDENT_OPTION(BUILD_STATIC_LIBS "Build static libraries" OFF "BUILD_SHARED_LIBS" ON) + +if(NOT BUILD_SHARED_LIBS AND NOT BUILD_STATIC_LIBS) + message(FATAL_ERROR "Both BUILD_SHARED_LIBS and BUILD_STATIC_LIBS have been disabled") +endif() set(LZ4_LIB_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/lib") set(LZ4_PROG_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/programs") @@ -90,28 +95,41 @@ set(LZ4_CLI_SOURCES # we're building a shared library this is ignored and PIC is always # used. option(LZ4_POSITION_INDEPENDENT_LIB "Use position independent code for static library (if applicable)" ON) -if(LZ4_POSITION_INDEPENDENT_LIB OR BUILD_SHARED_LIBS) - set(LZ4_POSITION_INDEPENDENT_CODE TRUE) -else() - set(LZ4_POSITION_INDEPENDENT_CODE FALSE) -endif() # liblz4 -add_library(lz4 ${LZ4_SOURCES}) -set_target_properties(lz4 PROPERTIES - SOVERSION "${LZ4_VERSION_STRING}" - VERSION "${LZ4_VERSION_STRING}" - POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_CODE}) +set(LZ4_LIBRARIES_BUILT) +if(BUILD_SHARED_LIBS) + add_library(lz4_shared SHARED ${LZ4_SOURCES}) + set_target_properties(lz4_shared PROPERTIES + OUTPUT_NAME lz4 + SOVERSION "${LZ4_VERSION_MAJOR}" + VERSION "${LZ4_VERSION_STRING}") + list(APPEND LZ4_LIBRARIES_BUILT lz4_shared) +endif() +if(BUILD_STATIC_LIBS) + add_library(lz4_static STATIC ${LZ4_SOURCES}) + set_target_properties(lz4_static PROPERTIES + OUTPUT_NAME lz4 + POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_LIB}) + list(APPEND LZ4_LIBRARIES_BUILT lz4_static) +endif() + +# link to shared whenever possible, to static otherwise +if(BUILD_SHARED_LIBS) + set(LZ4_LINK_LIBRARY lz4_shared) +else() + set(LZ4_LINK_LIBRARY lz4_static) +endif() # lz4 add_executable(lz4cli ${LZ4_CLI_SOURCES}) set_target_properties(lz4cli PROPERTIES OUTPUT_NAME lz4) -target_link_libraries(lz4cli lz4) +target_link_libraries(lz4cli ${LZ4_LINK_LIBRARY}) # lz4c add_executable(lz4c ${LZ4_CLI_SOURCES}) set_target_properties(lz4c PROPERTIES COMPILE_DEFINITIONS "ENABLE_LZ4C_LEGACY_OPTIONS") -target_link_libraries(lz4c lz4) +target_link_libraries(lz4c ${LZ4_LINK_LIBRARY}) # Extra warning flags include (CheckCCompilerFlag) @@ -149,7 +167,7 @@ if(NOT LZ4_BUNDLED_MODE) install(TARGETS lz4cli lz4c RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") - install(TARGETS lz4 + install(TARGETS ${LZ4_LIBRARIES_BUILT} LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") install(FILES @@ -157,8 +175,29 @@ if(NOT LZ4_BUNDLED_MODE) "${LZ4_LIB_SOURCE_DIR}/lz4frame.h" "${LZ4_LIB_SOURCE_DIR}/lz4hc.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + install(FILES "${LZ4_PROG_SOURCE_DIR}/lz4.1" + DESTINATION "${CMAKE_INSTALL_MANDIR}/man1") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblz4.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") + + # install lz4cat and unlz4 symlinks on *nix + if(UNIX) + install(CODE " + foreach(f lz4cat unlz4) + set(dest \"\$ENV{DESTDIR}${CMAKE_INSTALL_FULL_BINDIR}/\${f}\") + message(STATUS \"Symlinking: \${dest} -> lz4\") + execute_process( + COMMAND \"${CMAKE_COMMAND}\" -E create_symlink lz4 \"\${dest}\") + endforeach() + ") + + # create manpage aliases + foreach(f lz4cat unlz4) + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${f}.1" ".so man1/lz4.1\n") + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${f}.1" + DESTINATION "${CMAKE_INSTALL_MANDIR}/man1") + endforeach() + endif(UNIX) endif(NOT LZ4_BUNDLED_MODE) # pkg-config diff --git a/lib/Makefile b/lib/Makefile index 2b47fb0..2d9c8f3 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -42,11 +42,6 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT)) BUILD_STATIC:= yes -DESTDIR ?= -PREFIX ?= /usr/local -LIBDIR ?= $(PREFIX)/lib -INCLUDEDIR=$(PREFIX)/include - CPPFLAGS+= -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 DEBUGFLAGS:=-g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ @@ -71,6 +66,8 @@ else SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif +LIBLZ4 = liblz4.$(SHARED_EXT_VER) + default: lib-release lib-release: liblz4.a liblz4 @@ -90,27 +87,50 @@ ifeq ($(BUILD_STATIC),yes) @$(AR) rcs $@ *.o endif -liblz4: *.c +$(LIBLZ4): *.c @echo compiling dynamic library $(LIBVER) ifneq (,$(filter Windows%,$(OS))) @$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib else - @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) + @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@ @echo creating versioned links - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR) - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT) + @ln -sf $@ liblz4.$(SHARED_EXT_MAJOR) + @ln -sf $@ liblz4.$(SHARED_EXT) endif +liblz4: $(LIBLZ4) + clean: - @$(RM) core *.o *.a *.$(SHARED_EXT) liblz4.pc dll/liblz4.dll dll/liblz4.lib + @$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib + @$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER) @echo Cleaning library completed -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#----------------------------------------------------------------------------- +# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) + +ifneq (,$(filter $(shell uname),SunOS)) +INSTALL ?= ginstall +else +INSTALL ?= install +endif + +PREFIX ?= /usr/local +DESTDIR ?= +LIBDIR ?= $(PREFIX)/lib +INCLUDEDIR ?= $(PREFIX)/include + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly)) +PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig +else +PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig +endif + +INSTALL_LIB ?= $(INSTALL) -m 755 +INSTALL_DATA ?= $(INSTALL) -m 644 liblz4.pc: liblz4.pc.in Makefile @echo creating pkgconfig @@ -121,23 +141,25 @@ liblz4.pc: liblz4.pc.in Makefile $< >$@ install: lib liblz4.pc - @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ - @install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) - @install -m 644 liblz4.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ + @$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ + @$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/ + @echo Installing libraries ifeq ($(BUILD_STATIC),yes) - @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a + @$(INSTALL_LIB) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a endif - @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h - @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h + @$(INSTALL_LIB) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) + @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) + @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) + @echo Installing includes + @$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h + @$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h @echo lz4 static and shared libraries installed uninstall: + @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a @$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h diff --git a/lib/README.md b/lib/README.md index 18793d4..b40442c 100644 --- a/lib/README.md +++ b/lib/README.md @@ -47,7 +47,7 @@ The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library `dll\liblz4.dll` are required to compile a project using gcc/MinGW. The dynamic library has to be added to linking options. It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be compiled with "liblz4.lib". For example: +file it should be linked with `dll\liblz4.dll`. For example: ``` gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll ``` diff --git a/lib/dll/example/README.md b/lib/dll/example/README.md index 4908663..223e473 100644 --- a/lib/dll/example/README.md +++ b/lib/dll/example/README.md @@ -39,7 +39,7 @@ The header files from `include\` and the dynamic library `dll\liblz4.dll` are required to compile a project using gcc/MinGW. The dynamic library has to be added to linking options. It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be compiled with "liblz4.dll". For example: +file it should be linked with `dll\liblz4.dll`. For example: ``` gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\liblz4.dll ``` @@ -391,6 +391,7 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive; * Local Utils **************************************/ int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } int LZ4_sizeofState() { return LZ4_STREAMSIZE; } @@ -85,7 +85,7 @@ extern "C" { /*========== Version =========== */ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 3c2b788..a0a625b 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -158,7 +158,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB static const size_t minFHSize = 7; -static const size_t maxFHSize = 15; /* max Frame Header Size */ +static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 15 */ static const size_t BHSize = 4; @@ -254,20 +254,27 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI return requestedBSID; } +/* LZ4F_compressBound() : + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + */ static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered) { LZ4F_preferences_t prefsNull; memset(&prefsNull, 0, sizeof(prefsNull)); prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + U32 const flush = prefsPtr->autoFlush | (srcSize==0); LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; size_t const blockSize = LZ4F_getBlockSize(bid); size_t const maxBuffered = blockSize - 1; size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); size_t const maxSrcSize = srcSize + bufferedSize; unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); - size_t const partialBlockSize = srcSize & (blockSize-1); - size_t const lastBlockSize = prefsPtr->autoFlush ? partialBlockSize : 0; + size_t const partialBlockSize = (srcSize - (srcSize==0)) & (blockSize-1); /* 0 => -1 == MAX => blockSize-1 */ + size_t const lastBlockSize = flush ? partialBlockSize : 0; unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */ @@ -322,7 +329,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (prefs.frameInfo.contentSize != 0) prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - if (prefs.compressionLevel < LZ4HC_MIN_CLEVEL) { + if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { cctxI.lz4CtxPtr = &lz4ctx; cctxI.lz4CtxLevel = 1; } @@ -349,7 +356,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (LZ4F_isError(tailSize)) return tailSize; dstPtr += tailSize; } - if (prefs.compressionLevel >= LZ4HC_MIN_CLEVEL) /* no allocation done with lz4 fast */ + if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* no allocation done with lz4 fast */ FREEMEM(cctxI.lz4CtxPtr); return (dstPtr - dstStart); @@ -398,7 +405,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. + * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. * @return : number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ @@ -417,10 +424,10 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit cctxPtr->prefs = *preferencesPtr; /* ctx Management */ - { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ + { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ if (cctxPtr->lz4CtxLevel < tableID) { FREEMEM(cctxPtr->lz4CtxPtr); - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); else cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); @@ -445,7 +452,7 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; XXH32_reset(&(cctxPtr->xxh), 0); - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); else LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); @@ -526,7 +533,7 @@ static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) { - if (level < LZ4HC_MIN_CLEVEL) { + if (level < LZ4HC_CLEVEL_MIN) { if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; return LZ4F_localLZ4_compress_limitedOutput_continue; } @@ -536,7 +543,7 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) { - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); } diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 3104d2e..a4a4ce6 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -206,10 +206,11 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /* Compression */ +#define LZ4F_HEADER_SIZE_MAX 15 LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr); /* LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header. Maximum header size is 15 bytes. + * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. * @return : number of bytes written into dstBuffer for the header * or an error code (which can be tested using LZ4F_isError()) @@ -217,20 +218,20 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t d LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); /* LZ4F_compressBound() : - * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. - * Different preferences can produce different results. - * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. - * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. */ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); /* LZ4F_compressUpdate() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * An important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. - * This value is provided by using LZ4F_compressBound(). + * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. + * This value is provided by LZ4F_compressBound(). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). * LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) */ @@ -245,12 +246,12 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapaci * or an error code if it fails (which can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr); +LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); /* LZ4F_compressEnd() : - * To properly finish the compressed frame, invoke LZ4F_compressEnd(). + * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). * It will flush whatever data remained within `cctx` (like LZ4_flush()) * and properly finalize the frame, with an endMark and a checksum. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) * or an error code if it fails (which can be tested using LZ4F_isError()) * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 298550c..5d4ea3e 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -84,7 +84,7 @@ * Local Macros **************************************/ #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) -/* #define DELTANEXTU16(p) chainTable[(p) & LZ4HC_MAXD_MASK] */ /* flexible, LZ4HC_MAXD dependent */ +#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } @@ -305,14 +305,15 @@ FORCE_INLINE int LZ4HC_encodeSequence ( return 0; } +#include "lz4opt.h" -static int LZ4HC_compress_generic ( +static int LZ4HC_compress_hashChain ( LZ4HC_CCtx_internal* const ctx, const char* const source, char* const dest, int const inputSize, int const maxOutputSize, - int compressionLevel, + unsigned maxNbAttempts, limitedOutput_directive limit ) { @@ -325,7 +326,6 @@ static int LZ4HC_compress_generic ( BYTE* op = (BYTE*) dest; BYTE* const oend = op + maxOutputSize; - unsigned maxNbAttempts; int ml, ml2, ml3, ml0; const BYTE* ref = NULL; const BYTE* start2 = NULL; @@ -336,9 +336,6 @@ static int LZ4HC_compress_generic ( const BYTE* ref0; /* init */ - if (compressionLevel > LZ4HC_MAX_CLEVEL) compressionLevel = LZ4HC_MAX_CLEVEL; - if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; - maxNbAttempts = 1 << (compressionLevel-1); ctx->end += inputSize; ip++; @@ -489,6 +486,37 @@ _Search3: return (int) (((char*)op)-dest); } +static int LZ4HC_getSearchNum(int compressionLevel) +{ + switch (compressionLevel) { + default: return 0; /* unused */ + case 11: return 128; + case 12: return 1<<10; + } +} + +static int LZ4HC_compress_generic ( + LZ4HC_CCtx_internal* const ctx, + const char* const source, + char* const dest, + int const inputSize, + int const maxOutputSize, + int compressionLevel, + limitedOutput_directive limit + ) +{ + if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; + if (compressionLevel > 9) { + switch (compressionLevel) { + case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); + case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); + default: + case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); + } + } + return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit); +} + int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } @@ -534,6 +562,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel; + LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel); } int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) @@ -544,8 +573,11 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize = 64 KB; } LZ4HC_init (ctxPtr, (const BYTE*)dictionary); - if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3)); ctxPtr->end = (const BYTE*)dictionary + dictSize; + if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) + LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); + else + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); return dictSize; } @@ -554,7 +586,11 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { - if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) + LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); + else + if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); diff --git a/lib/lz4hc.h b/lib/lz4hc.h index d8ac3c0..1036fd0 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -44,9 +44,10 @@ extern "C" { /* --- Useful constants --- */ -#define LZ4HC_MIN_CLEVEL 3 -#define LZ4HC_DEFAULT_CLEVEL 9 -#define LZ4HC_MAX_CLEVEL 16 +#define LZ4HC_CLEVEL_MIN 3 +#define LZ4HC_CLEVEL_DEFAULT 9 +#define LZ4HC_CLEVEL_OPT_MIN 11 +#define LZ4HC_CLEVEL_MAX 12 /*-************************************ @@ -58,7 +59,7 @@ extern "C" { * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work. - * Values >LZ4HC_MAX_CLEVEL behave the same as 16. + * Values >LZ4HC_MAX_CLEVEL behave the same as LZ4HC_MAX_CLEVEL. * @return : the number of bytes written into 'dst' * or 0 if compression fails. */ @@ -130,11 +131,11 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in * They are exposed to allow static allocation of `LZ4_streamHC_t`. * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 **************************************/ -#define LZ4HC_DICTIONARY_LOGSIZE 16 +#define LZ4HC_DICTIONARY_LOGSIZE 17 #define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE) #define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) -#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) +#define LZ4HC_HASH_LOG 15 #define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG) #define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) @@ -153,6 +154,7 @@ typedef struct uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ + uint32_t searchNum; /* only for optimal parser */ uint32_t compressionLevel; } LZ4HC_CCtx_internal; @@ -169,12 +171,13 @@ typedef struct unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ + unsigned int searchNum; /* only for optimal parser */ unsigned int compressionLevel; } LZ4HC_CCtx_internal; #endif -#define LZ4_STREAMHCSIZE 262192 +#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 393268 */ #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) union LZ4_streamHC_u { size_t table[LZ4_STREAMHCSIZE_SIZET]; diff --git a/lib/lz4opt.h b/lib/lz4opt.h new file mode 100644 index 0000000..d1913fe --- /dev/null +++ b/lib/lz4opt.h @@ -0,0 +1,363 @@ +/* + lz4opt.h - Optimal Mode of LZ4 + Copyright (C) 2015-2016, Przemyslaw Skibinski <inikep@gmail.com> + Note : this file is intended to be included within lz4hc.c + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +#define LZ4_OPT_NUM (1<<12) + + +typedef struct +{ + int off; + int len; +} LZ4HC_match_t; + +typedef struct +{ + int price; + int off; + int mlen; + int litlen; +} LZ4HC_optimal_t; + + +/* price in bits */ +FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen) +{ + size_t price = 8*litlen; + if (litlen >= (size_t)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); + return price; +} + + +/* requires mlen >= MINMATCH */ +FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen) +{ + size_t price = 16 + 8; /* 16-bit offset + token */ + + price += LZ4HC_literalsPrice(litlen); + + mlen -= MINMATCH; + if (mlen >= (size_t)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255); + + return price; +} + + +/*-************************************* +* Binary Tree search +***************************************/ +FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( + LZ4HC_CCtx_internal* ctx, + const BYTE* const ip, + const BYTE* const iHighLimit, + size_t best_mlen, + LZ4HC_match_t* matches, + int* matchNum) +{ + U16* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1); + const BYTE* const dictBase = ctx->dictBase; + const BYTE* match; + int nbAttempts = ctx->searchNum; + int mnum = 0; + U16 *ptr0, *ptr1, delta0, delta1; + U32 matchIndex; + size_t matchLength = 0; + U32* HashPos; + + if (ip + MINMATCH > iHighLimit) return 1; + + /* HC4 match finder */ + HashPos = &HashTable[LZ4HC_hashPtr(ip)]; + matchIndex = *HashPos; + *HashPos = current; + + ptr0 = &DELTANEXTMAXD(current*2+1); + ptr1 = &DELTANEXTMAXD(current*2); + delta0 = delta1 = (U16)(current - matchIndex); + + while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { + nbAttempts--; + if (matchIndex >= dictLimit) { + match = base + matchIndex; + matchLength = LZ4_count(ip, match, iHighLimit); + } else { + const BYTE* vLimit = ip + (dictLimit - matchIndex); + match = dictBase + matchIndex; + if (vLimit > iHighLimit) vLimit = iHighLimit; + matchLength = LZ4_count(ip, match, vLimit); + if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) + matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit); + } + + if (matchLength > best_mlen) { + best_mlen = matchLength; + if (matches) { + if (matchIndex >= dictLimit) + matches[mnum].off = (int)(ip - match); + else + matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */ + matches[mnum].len = (int)matchLength; + mnum++; + } + if (best_mlen > LZ4_OPT_NUM) break; + } + + if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ + + if (*(ip+matchLength) < *(match+matchLength)) { + *ptr0 = delta0; + ptr0 = &DELTANEXTMAXD(matchIndex*2); + if (*ptr0 == (U16)-1) break; + delta0 = *ptr0; + delta1 += delta0; + matchIndex -= delta0; + } else { + *ptr1 = delta1; + ptr1 = &DELTANEXTMAXD(matchIndex*2+1); + if (*ptr1 == (U16)-1) break; + delta1 = *ptr1; + delta0 += delta1; + matchIndex -= delta1; + } + } + + *ptr0 = (U16)-1; + *ptr1 = (U16)-1; + if (matchNum) *matchNum = mnum; + /* if (best_mlen > 8) return best_mlen-8; */ + if (!matchNum) return 1; + return 1; +} + + +FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit) +{ + const BYTE* const base = ctx->base; + const U32 target = (U32)(ip - base); + U32 idx = ctx->nextToUpdate; + while(idx < target) + idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); +} + + +/** Tree updater, providing best match */ +FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( + LZ4HC_CCtx_internal* ctx, + const BYTE* const ip, const BYTE* const iHighLimit, + size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate) +{ + int mnum = 0; + if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ + if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); + best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); + ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen); + return mnum; +} + + +#define SET_PRICE(pos, mlen, offset, litlen, price) \ +{ \ + while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ + opt[pos].mlen = (int)mlen; \ + opt[pos].off = (int)offset; \ + opt[pos].litlen = (int)litlen; \ + opt[pos].price = (int)price; \ +} + + +static int LZ4HC_compress_optimal ( + LZ4HC_CCtx_internal* ctx, + const char* const source, + char* dest, + int inputSize, + int maxOutputSize, + limitedOutput_directive limit, + const size_t sufficient_len, + const int fullUpdate + ) +{ + LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; + LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; + const BYTE *inr = NULL; + size_t res, cur, cur2; + size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + + /* init */ + ctx->end += inputSize; + ip++; + + /* Main Loop */ + while (ip < mflimit) { + memset(opt, 0, sizeof(LZ4HC_optimal_t)); + last_pos = 0; + llen = ip - anchor; + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); + if (!match_num) { ip++; continue; } + + if ((size_t)matches[match_num-1].len > sufficient_len) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto encode; + } + + /* set prices using matches at position = 0 */ + for (i = 0; i < match_num; i++) { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; + best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; + while (mlen <= best_mlen) { + litlen = 0; + price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); + SET_PRICE(mlen, mlen, matches[i].off, litlen, price); + mlen++; + } + } + + if (last_pos < MINMATCH) { ip++; continue; } + + /* check further positions */ + opt[0].mlen = opt[1].mlen = 1; + for (cur = 1; cur <= last_pos; cur++) { + inr = ip + cur; + + if (opt[cur-1].mlen == 1) { + litlen = opt[cur-1].litlen + 1; + if (cur != litlen) { + price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); + } else { + price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); + } + } else { + litlen = 1; + price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen); + } + + mlen = 1; + best_mlen = 0; + if (cur > last_pos || price < (size_t)opt[cur].price) + SET_PRICE(cur, mlen, best_mlen, litlen, price); + + if (cur == last_pos || inr >= mflimit) break; + + match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); + if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto encode; + } + + /* set prices using matches at position = cur */ + for (i = 0; i < match_num; i++) { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; + cur2 = cur; + best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; + + while (mlen <= best_mlen) { + if (opt[cur2].mlen == 1) { + litlen = opt[cur2].litlen; + + if (cur2 != litlen) + price = opt[cur2 - litlen].price + LZ4HC_sequencePrice(litlen, mlen); + else + price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); + } else { + litlen = 0; + price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen); + } + + if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) { + SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); + } + mlen++; + } + } + } /* for (cur = 1; cur <= last_pos; cur++) */ + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + +encode: /* cur, last_pos, best_mlen, best_off have to be set */ + opt[0].mlen = 1; + while (1) { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = (int)best_mlen; + opt[cur].off = (int)best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + cur = 0; + while (cur < last_pos) { + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + + res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); + if (res) return 0; + } + } + + /* Encode Last Literals */ + { int lastRun = (int)(iend - anchor); + if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun<<ML_BITS); + memcpy(op, anchor, iend - anchor); + op += iend-anchor; + } + + /* End */ + return (int) ((char*)op-dest); +} diff --git a/programs/.gitignore b/programs/.gitignore index df3423e..daa7f14 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -1,5 +1,7 @@ # local binary (Makefile) lz4 +unlz4 +lz4cat lz4c lz4c32 datagen diff --git a/programs/Makefile b/programs/Makefile index 504416f..060ce21 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -27,12 +27,21 @@ # lz4c32: Same as lz4c, but forced to compile in 32-bits mode # ########################################################################## -DESTDIR ?= -PREFIX ?= /usr/local -BINDIR := $(PREFIX)/bin -MANDIR := $(PREFIX)/share/man/man1 +# Version numbers +LIBVER_SRC := ../lib/lz4.h +LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) +LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) +LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) + LZ4DIR := ../lib -VOID := /dev/null + +SRCFILES := $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c) +OBJFILES := $(patsubst %.c,%.o,$(SRCFILES)) CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 @@ -42,68 +51,115 @@ DEBUGFLAGS:=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ CFLAGS += $(MOREFLAGS) FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) +LZ4_VERSION=$(LIBVER) +MD2ROFF =ronn +MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)" + # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) -EXT =.exe +VOID := nul +EXT :=.exe else -EXT = +VOID := /dev/null +EXT := endif default: lz4-release -lz4-release: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o - $(CC) $(FLAGS) $^ -o lz4$(EXT) - all: lz4 lz4c all32: CFLAGS+=-m32 all32: all lz4: CFLAGS += $(DEBUGFLAGS) -lz4: lz4-release +lz4: $(OBJFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) + +lz4-release: DEBUGFLAGS= +lz4-release: lz4 -lz4c: CFLAGS += $(DEBUGFLAGS) -lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.c datagen.o +lz4c : CFLAGS += $(DEBUGFLAGS) +lz4c : $(SRCFILES) $(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) -lz4c32: CFLAGS+=-m32 -lz4c32: lz4 - @cp lz4$(EXT) lz4c32$(EXT) +lz4c32: CFLAGS += -m32 $(DEBUGFLAGS) +lz4c32: $(SRCFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(RM) core *.o *.test tmp* \ - lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) + lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat @echo Cleaning completed -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +lz4.1: lz4.1.md + cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ + +man: lz4.1 + +clean-man: + rm lz4.1 + +preview-man: clean-man man + man ./lz4.1 + + +#----------------------------------------------------------------------------- +# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) + +unlz4: lz4 + ln -s lz4 unlz4 + +lz4cat: lz4 + ln -s lz4 lz4cat + + +ifneq (,$(filter $(shell uname),SunOS)) +INSTALL ?= ginstall +else +INSTALL ?= install +endif + +PREFIX ?= /usr/local +DESTDIR ?= +BINDIR ?= $(PREFIX)/bin + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) +MANDIR ?= $(PREFIX)/man/man1 +else +MANDIR ?= $(PREFIX)/share/man/man1 +endif + +INSTALL_PROGRAM ?= $(INSTALL) -m 755 +INSTALL_SCRIPT ?= $(INSTALL) -m 755 +INSTALL_MAN ?= $(INSTALL) -m 644 + install: lz4$(EXT) lz4c$(EXT) @echo Installing binaries - @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4$(EXT) - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4cat$(EXT) - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/unlz4$(EXT) - @install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c$(EXT) + @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ + @$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4 + @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat + @ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4 + @$(INSTALL_PROGRAM) lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c @echo Installing man pages - @install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 + @$(INSTALL_MAN) -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1 @echo lz4 installation completed uninstall: - @$(RM) $(DESTDIR)$(BINDIR)/lz4cat$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/unlz4$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/lz4$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/lz4c$(EXT) + @$(RM) $(DESTDIR)$(BINDIR)/lz4cat + @$(RM) $(DESTDIR)$(BINDIR)/unlz4 + @$(RM) $(DESTDIR)$(BINDIR)/lz4 + @$(RM) $(DESTDIR)$(BINDIR)/lz4c @$(RM) $(DESTDIR)$(MANDIR)/lz4.1 @$(RM) $(DESTDIR)$(MANDIR)/lz4c.1 @$(RM) $(DESTDIR)$(MANDIR)/lz4cat.1 diff --git a/programs/bench.c b/programs/bench.c index 434da8b..8311503 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -24,13 +24,22 @@ */ +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + /* ************************************* * Includes ***************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ #include <stdlib.h> /* malloc, free */ #include <string.h> /* memset */ -#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <stdio.h> /* fprintf, fopen, ftello */ #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ #include "datagen.h" /* RDG_genBuffer */ @@ -170,7 +179,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_initTimer(&ticksPerSecond); /* Init */ - if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1; + if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1; switch (cfunctionId) { #ifdef COMPRESSOR0 @@ -494,8 +503,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, { double const compressibility = (double)g_compressibilityDefault / 100; - if (cLevel > LZ4HC_MAX_CLEVEL) cLevel = LZ4HC_MAX_CLEVEL; - if (cLevelLast > LZ4HC_MAX_CLEVEL) cLevelLast = LZ4HC_MAX_CLEVEL; + if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; + if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); diff --git a/programs/datagen.c b/programs/datagen.c index f97c2d0..a61afc0 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -26,43 +26,14 @@ /************************************** * Includes **************************************/ +#include "platform.h" /* Compiler options, SET_BINARY_MODE */ +#include "util.h" /* U32 */ #include <stdlib.h> /* malloc */ #include <stdio.h> /* FILE, fwrite */ #include <string.h> /* memcpy */ /************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** -* OS-specific Includes -**************************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include <fcntl.h> /* _O_BINARY */ -# include <io.h> /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) -#else -# define SET_BINARY_MODE(file) -#endif - - -/************************************** * Constants **************************************/ #define KB *(1 <<10) diff --git a/programs/lz4.1 b/programs/lz4.1 index 2c94033..48c988a 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -1,251 +1,206 @@ -\." -\." lz4.1: This is a manual page for 'lz4' program. This file is part of the -\." lz4 <http://www.lz4.org/> project. -\." Author: Yann Collet -\." -. -\." No hyphenation -.hy 0 -.nr HY 0 -. -.TH lz4 "1" "2015-03-21" "lz4" "User Commands" -.SH NAME -\fBlz4, unlz4, lz4cat\fR \- Compress or decompress .lz4 files - -.SH SYNOPSIS -.TP 5 -\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE> -.PP -.B unlz4 -is equivalent to -.BR "lz4 \-d" -.br -.B lz4cat -is equivalent to -.BR "lz4 \-dcfm" -.br -.PP -When writing scripts that need to decompress files, -it is recommended to always use the name -.B lz4 -with appropriate arguments -.RB ( "lz4 \-d" -or -.BR "lz4 \-dc" ) -instead of the names -.B unlz4 -and -.BR lz4cat . - - -.SH DESCRIPTION -.PP -\fBlz4\fR is an extremely fast lossless compression algorithm, -based on \fBbyte-aligned LZ77\fR family of compression scheme. -\fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs. -It features an extremely fast decoder, with speed in multiple GB/s per core, -typically reaching RAM speed limit on multi-core systems. -The native file format is the -.B .lz4 -format. - -.B lz4 -supports a command line syntax similar \fIbut not identical\fR to -.BR gzip (1). -Differences are : - \fBlz4\fR preserves original files - \fBlz4\fR compresses a single file by default (use \fB-m\fR for multiple files) - \fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 - When no destination name is provided, compressed file name receives a \fB.lz4\fR suffix - When no destination name is provided, if \fBstdout\fR is \fInot\fR the console, it becomes the output (like a silent \fB-c\fR) - Therefore \fBlz4 file > /dev/null\fR will not create \fBfile.lz4\fR - \fBlz4 file\fR shows real-time statistics during compression (use \fB-q\fR to silent them) - -Default behaviors can be modified by opt-in commands, described below. -\fBlz4 --quiet --multiple\fR more closely mimics \fBgzip\fR behavior. - -.SS "Concatenation of .lz4 files" -It is possible to concatenate -.B .lz4 -files as is. -.B lz4 -will decompress such files as if they were a single -.B .lz4 -file. For example: - lz4 file1 > foo.lz4 - lz4 file2 >> foo.lz4 - then - lz4cat foo.lz4 - is equivalent to : - cat file1 file2 - -.PP - -.SH OPTIONS +. +.TH "LZ4" "1" "November 2016" "lz4 1.7.4" "User Commands" +. +.SH "NAME" +\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files +. +.SH "SYNOPSIS" +\fBlz4\fR [\fIOPTIONS\fR] [\-|INPUT\-FILE] \fIOUTPUT\-FILE\fR +. +.P +\fBunlz4\fR is equivalent to \fBlz4 \-d\fR +. +.P +\fBlz4cat\fR is equivalent to \fBlz4 \-dcfm\fR +. +.P +When writing scripts that need to decompress files, it is recommended to always use the name \fBlz4\fR with appropriate arguments (\fBlz4 \-d\fR or \fBlz4 \-dc\fR) instead of the names \fBunlz4\fR and \fBlz4cat\fR\. +. +.SH "DESCRIPTION" +\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\. +. +.SS "Difference between lz4 and gzip" +\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are : +. +.IP "\(bu" 4 +\fBlz4\fR preserves original files +. +.IP "\(bu" 4 +\fBlz4\fR compresses a single file by default (see \fB\-m\fR for multiple files) +. +.IP "\(bu" 4 +\fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 +. +.IP "\(bu" 4 +\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silent them) +. +.IP "\(bu" 4 +If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\. +. +.IP "\(bu" 4 +If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\. +. +.IP "\(bu" 4 +As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create any \fBfile\.lz4\fR\. +. +.IP "" 0 +. +.P +Default behaviors can be modified by opt\-in commands, detailed below\. +. +.IP "\(bu" 4 +\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main difference being that source files are preserved by default\. +. +.IP "\(bu" 4 +It\'s possible to opt\-in to erase source files on successful compression or decompression, using \fB\-\-rm\fR command\. +. +.IP "\(bu" 4 +Consequently, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\. +. +.IP "" 0 +. +.SS "Concatenation of \.lz4 files" +It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example: lz4 file1 > foo\.lz4 lz4 file2 >> foo\.lz4 +. +.P +then lz4cat foo\.lz4 +. +.P +is equivalent to : cat file1 file2 +. +.SH "OPTIONS" . .SS "Short commands concatenation" -In some cases, some options can be expressed using short command -.B "-x" -or long command -.B "--long-word" . -Short commands can be concatenated together. For example, -.B "-d -c" -is equivalent to -.B "-dc" . -Long commands cannot be concatenated. -They must be clearly separated by a space. +In some cases, some options can be expressed using short command \fB\-x\fR or long command \fB\-\-long\-word\fR\. Short commands can be concatenated together\. For example, \fB\-d \-c\fR is equivalent to \fB\-dc\fR\. Long commands cannot be concatenated\. They must be clearly separated by a space\. +. .SS "Multiple commands" -When multiple contradictory commands are issued on a same command line, -only the latest one will be applied. +When multiple contradictory commands are issued on a same command line, only the latest one will be applied\. . .SS "Operation mode" +. .TP -.BR \-z ", " \-\-compress -Compress. -This is the default operation mode -when no operation mode option is specified , -no other operation mode is implied from the command name -(for example, -.B unlz4 -implies -.B \-\-decompress ), -nor from the input file name -(for example, a file extension -.B .lz4 -implies -.B \-\-decompress -by default). -.B -z -can also be used to force compression of an already compressed -.B .lz4 -file. -.TP -.BR \-d ", " \-\-decompress ", " \-\-uncompress -Decompress. -.B --decompress -is also the default operation when the input filename has an -.B .lz4 -extension. -.TP -.BR \-t ", " \-\-test -Test the integrity of compressed -.B .lz4 -files. -The decompressed data is discarded. -No files are created nor removed. - -.TP -.BR \-b# -Benchmark mode, using # compression level. - +\fB\-z\fR \fB\-\-compress\fR +Compress\. This is the default operation mode when no operation mode option is specified, no other operation mode is implied from the command name (for example, \fBunlz4\fR implies \fB\-\-decompress\fR), nor from the input file name (for example, a file extension \fB\.lz4\fR implies \fB\-\-decompress\fR by default)\. \fB\-z\fR can also be used to force compression of an already compressed \fB\.lz4\fR file\. +. +.TP +\fB\-d\fR \fB\-\-decompress\fR \fB\-\-uncompress\fR +Decompress\. \fB\-\-decompress\fR is also the default operation when the input filename has an \fB\.lz4\fR extension\. +. +.TP +\fB\-t\fR \fB\-\-test\fR +Test the integrity of compressed \fB\.lz4\fR files\. The decompressed data is discarded\. No files are created nor removed\. +. +.TP +\fB\-b#\fR +Benchmark mode, using \fB#\fR compression level\. . .SS "Operation modifiers" +. +.TP +\fB\-#\fR +Compression level, with # being any value from 1 to 16\. Higher values trade compression speed for compression ratio\. Values above 16 are considered the same as 16\. Recommended values are 1 for fast compression (default), and 9 for high compression\. Speed/compression trade\-off will vary depending on data to compress\. Decompression speed remains fast at all settings\. +. .TP -.B \-# - compression level, with # being any value from 1 to 16. - Higher values trade compression speed for compression ratio. - Values above 16 are considered the same as 16. - Recommended values are 1 for fast compression (default), and 9 for high compression. - Speed/compression trade-off will vary depending on data to compress. - Decompression speed remains fast at all settings. - -.TP -.BR \-f ", " --[no-]force - This option has several effects: -.RS -.IP \(bu 3 -If the target file already exists, -overwrite it without prompting. -.IP \(bu 3 -When used with -.B \-\-decompress -and -.B lz4 -cannot recognize the type of the source file, -copy the source file as is to standard output. -This allows -.B lz4cat -.B \-\-force -to be used like -.BR cat (1) -for files that have not been compressed with -.BR lz4 . -.RE - -.TP -.BR \-c ", " \--stdout ", " \--to-stdout - force write to standard output, even if it is the console - -.TP -.BR \-m ", " \--multiple - Multiple file names. - By default, the second filename is used as the destination filename for the compressed file. - With -.B -m -, it is possible to specify any number of input filenames. Each of them will be compressed -independently, and the resulting name of each compressed file will be -.B filename.lz4 -. - -.TP -.B \-B# - block size [4-7](default : 7) - B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB -.TP -.B \-BD - block dependency (improves compression ratio on small blocks) -.TP -.B \--[no-]frame-crc - select frame checksum (default:enabled) -.TP -.B \--[no-]content-size - header includes original size (default:not present) - Note : this option can only be activated when the original size can be determined, -hence for a file. It won't work with unknown source size, such as stdin or pipe. -.TP -.B \--[no-]sparse - sparse mode support (default:enabled on file, disabled on stdout) -.TP -.B \-l - use Legacy format (typically used for Linux Kernel compression) - note : \fB-l\fR is not compatible with \fB-m\fR (\fB--multiple\fR) +\fB\-f\fR \fB\-\-[no\-]force\fR +This option has several effects: +. +.IP +If the target file already exists, overwrite it without prompting\. +. +.IP +When used with \fB\-\-decompress\fR and \fBlz4\fR cannot recognize the type of the source file, copy the source file as is to standard output\. This allows \fBlz4cat \-\-force\fR to be used like \fBcat (1)\fR for files that have not been compressed with \fBlz4\fR\. +. +.TP +\fB\-c\fR \fB\-\-stdout\fR \fB\-\-to\-stdout\fR +Force write to standard output, even if it is the console\. +. +.TP +\fB\-m\fR \fB\-\-multiple\fR +Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\. +. +.TP +\fB\-r\fR +operate recursively on directories\. This mode also sets \fB\-m\fR (multiple input files)\. +. +.TP +\fB\-B#\fR +Block size [4\-7](default : 7) +. +.br +\fB\-B4\fR= 64KB ; \fB\-B5\fR= 256KB ; \fB\-B6\fR= 1MB ; \fB\-B7\fR= 4MB +. +.TP +\fB\-BD\fR +Block Dependency (improves compression ratio on small blocks) +. +.TP +\fB\-\-[no\-]frame\-crc\fR +Select frame checksum (default:enabled) +. +.TP +\fB\-\-[no\-]content\-size\fR +Header includes original size (default:not present) +. +.br +Note : this option can only be activated when the original size can be determined, hence for a file\. It won\'t work with unknown source size, such as stdin or pipe\. +. +.TP +\fB\-\-[no\-]sparse\fR +Sparse mode support (default:enabled on file, disabled on stdout) +. +.TP +\fB\-l\fR +Use Legacy format (typically for Linux Kernel compression) +. +.br +Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) nor \fB\-r\fR . .SS "Other options" +. .TP -.BR \-v ", " --verbose - verbose mode +\fB\-v\fR \fB\-\-verbose\fR +Verbose mode +. .TP -.BR \-q ", " --quiet - suppress warnings and real-time statistics; specify twice to suppress errors too +\fB\-q\fR \fB\-\-quiet\fR +Suppress warnings and real\-time statistics; specify twice to suppress errors too +. .TP -.B \-h/\-H ", " --help - display help/long help and exit +\fB\-h\fR \fB\-H\fR \fB\-\-help\fR +Display help/long help and exit +. .TP -.BR \-V ", " \--version - display Version number and exit +\fB\-V\fR \fB\-\-version\fR +Display Version number and exit +. +.TP +\fB\-k\fR \fB\-\-keep\fR +Preserve source files (default behavior) +. .TP -.BR \-k ", " \--keep - Don't delete source file. -This is default behavior anyway, so this option is just for compatibility with gzip/xz. - +\fB\-\-rm\fR +Delete source files on successful compression or decompression . .SS "Benchmark mode" +. .TP -.B \-b# - benchmark file(s), using # compression level +\fB\-b#\fR +Benchmark file(s), using # compression level +. .TP -.B \-e# - benchmark multiple compression levels, from b# to e# (included) +\fB\-e#\fR +Benchmark multiple compression levels, from b# to e# (included) +. .TP -.B \-i# - minimum evaluation in seconds [1-9] (default : 3) +\fB\-i#\fR +Minimum evaluation in seconds [1\-9] (default : 3) +. .TP -.B \-r - operate recursively on directories - - -.SH BUGS -Report bugs at: https://github.com/Cyan4973/lz4/issues - -.SH AUTHOR +\fB\-r\fR +Operate recursively on directories +. +.SH "BUGS" +Report bugs at: https://github\.com/lz4/lz4/issues +. +.SH "AUTHOR" Yann Collet diff --git a/programs/lz4.1.md b/programs/lz4.1.md new file mode 100644 index 0000000..48f3152 --- /dev/null +++ b/programs/lz4.1.md @@ -0,0 +1,219 @@ +lz4(1) -- lz4, unlz4, lz4cat - Compress or decompress .lz4 files +================================================================ + +SYNOPSIS +-------- + +`lz4` [*OPTIONS*] [-|INPUT-FILE] <OUTPUT-FILE> + +`unlz4` is equivalent to `lz4 -d` + +`lz4cat` is equivalent to `lz4 -dcfm` + +When writing scripts that need to decompress files, +it is recommended to always use the name `lz4` with appropriate arguments +(`lz4 -d` or `lz4 -dc`) instead of the names `unlz4` and `lz4cat`. + + +DESCRIPTION +----------- + +`lz4` is an extremely fast lossless compression algorithm, +based on **byte-aligned LZ77** family of compression scheme. +`lz4` offers compression speeds of 400 MB/s per core, linearly scalable with +multi-core CPUs. +It features an extremely fast decoder, with speed in multiple GB/s per core, +typically reaching RAM speed limit on multi-core systems. +The native file format is the `.lz4` format. + +### Difference between lz4 and gzip + +`lz4` supports a command line syntax similar _but not identical_ to `gzip(1)`. +Differences are : + + * `lz4` preserves original files + * `lz4` compresses a single file by default (see `-m` for multiple files) + * `lz4 file1 file2` means : compress file1 _into_ file2 + * `lz4 file.lz4` will default to decompression (use `-z` to force compression) + * `lz4` shows real-time notification statistics + during compression or decompression of a single file + (use `-q` to silent them) + * If no destination name is provided, result is sent to `stdout` + _except if stdout is the console_. + * If no destination name is provided, __and__ if `stdout` is the console, + `file` is compressed into `file.lz4`. + * As a consequence of previous rules, note the following example : + `lz4 file | consumer` sends compressed data to `consumer` through `stdout`, + hence it does _not_ create `file.lz4`. + +Default behaviors can be modified by opt-in commands, detailed below. + + * `lz4 -m` makes it possible to provide multiple input filenames, + which will be compressed into files using suffix `.lz4`. + Progress notifications are also disabled by default (use `-v` to enable them). + This mode has a behavior which more closely mimics `gzip` command line, + with the main remaining difference being that source files are preserved by default. + * Similarly, `lz4 -m -d` can decompress multiple `*.lz4` files. + * It's possible to opt-in to erase source files + on successful compression or decompression, using `--rm` command. + * Consequently, `lz4 -m --rm` behaves the same as `gzip`. + +### Concatenation of .lz4 files + +It is possible to concatenate `.lz4` files as is. +`lz4` will decompress such files as if they were a single `.lz4` file. +For example: + lz4 file1 > foo.lz4 + lz4 file2 >> foo.lz4 + +then + lz4cat foo.lz4 + +is equivalent to : + cat file1 file2 + + +OPTIONS +------- + +### Short commands concatenation + +In some cases, some options can be expressed using short command `-x` +or long command `--long-word`. +Short commands can be concatenated together. +For example, `-d -c` is equivalent to `-dc`. +Long commands cannot be concatenated. +They must be clearly separated by a space. + +### Multiple commands + +When multiple contradictory commands are issued on a same command line, +only the latest one will be applied. + +### Operation mode + +* `-z` `--compress`: + Compress. + This is the default operation mode when no operation mode option is + specified, no other operation mode is implied from the command name + (for example, `unlz4` implies `--decompress`), + nor from the input file name + (for example, a file extension `.lz4` implies `--decompress` by default). + `-z` can also be used to force compression of an already compressed + `.lz4` file. + +* `-d` `--decompress` `--uncompress`: + Decompress. + `--decompress` is also the default operation when the input filename has an + `.lz4` extension. + +* `-t` `--test`: + Test the integrity of compressed `.lz4` files. + The decompressed data is discarded. + No files are created nor removed. + +* `-b#`: + Benchmark mode, using `#` compression level. + +### Operation modifiers + +* `-#`: + Compression level, with # being any value from 1 to 16. + Higher values trade compression speed for compression ratio. + Values above 16 are considered the same as 16. + Recommended values are 1 for fast compression (default), + and 9 for high compression. + Speed/compression trade-off will vary depending on data to compress. + Decompression speed remains fast at all settings. + +* `-f` `--[no-]force`: + This option has several effects: + + If the target file already exists, overwrite it without prompting. + + When used with `--decompress` and `lz4` cannot recognize the type of + the source file, copy the source file as is to standard output. + This allows `lz4cat --force` to be used like `cat (1)` for files + that have not been compressed with `lz4`. + +* `-c` `--stdout` `--to-stdout`: + Force write to standard output, even if it is the console. + +* `-m` `--multiple`: + Multiple input files. + Compressed file names will be appended a `.lz4` suffix. + This mode also reduces notification level. + `lz4 -m` has a behavior equivalent to `gzip -k` + (it preserves source files by default). + +* `-r` : + operate recursively on directories. + This mode also sets `-m` (multiple input files). + +* `-B#`: + Block size \[4-7\](default : 7)<br/> + `-B4`= 64KB ; `-B5`= 256KB ; `-B6`= 1MB ; `-B7`= 4MB + +* `-BD`: + Block Dependency (improves compression ratio on small blocks) + +* `--[no-]frame-crc`: + Select frame checksum (default:enabled) + +* `--[no-]content-size`: + Header includes original size (default:not present)<br/> + Note : this option can only be activated when the original size can be + determined, hence for a file. It won't work with unknown source size, + such as stdin or pipe. + +* `--[no-]sparse`: + Sparse mode support (default:enabled on file, disabled on stdout) + +* `-l`: + Use Legacy format (typically for Linux Kernel compression)<br/> + Note : `-l` is not compatible with `-m` (`--multiple`) nor `-r` + +### Other options + +* `-v` `--verbose`: + Verbose mode + +* `-q` `--quiet`: + Suppress warnings and real-time statistics; + specify twice to suppress errors too + +* `-h` `-H` `--help`: + Display help/long help and exit + +* `-V` `--version`: + Display Version number and exit + +* `-k` `--keep`: + Preserve source files (default behavior) + +* `--rm` : + Delete source files on successful compression or decompression + + +### Benchmark mode + +* `-b#`: + Benchmark file(s), using # compression level + +* `-e#`: + Benchmark multiple compression levels, from b# to e# (included) + +* `-i#`: + Minimum evaluation in seconds \[1-9\] (default : 3) + + +BUGS +---- + +Report bugs at: https://github.com/lz4/lz4/issues + + +AUTHOR +------ + +Yann Collet diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 71378aa..cf91a99 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -39,17 +39,11 @@ /* #define ENABLE_LZ4C_LEGACY_OPTIONS */ -/************************************** -* Compiler Options -***************************************/ -/* cf. http://man7.org/linux/man-pages/man7/feature_test_macros.7.html */ -#define _XOPEN_VERSION 600 /* POSIX.2001, for fileno() within <stdio.h> on unix */ - - /**************************** * Includes *****************************/ -#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ +#include "platform.h" /* Compiler options, IS_CONSOLE */ +#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ #include <stdio.h> /* fprintf, getchar */ #include <stdlib.h> /* exit, calloc, free */ #include <string.h> /* strcmp, strlen */ @@ -59,26 +53,12 @@ #include "lz4.h" /* LZ4_VERSION_STRING */ -/*-************************************ -* OS-specific Includes -**************************************/ -#if defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || defined(__DJGPP__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ -# include <unistd.h> /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include <io.h> /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -# define IS_CONSOLE(stdStream) 0 -#endif - - /***************************** * Constants ******************************/ #define COMPRESSOR_NAME "LZ4 command line interface" #define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION_STRING, AUTHOR +#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_versionString(), AUTHOR #define LZ4_EXTENSION ".lz4" #define LZ4CAT "lz4cat" #define UNLZ4 "unlz4" @@ -129,19 +109,20 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output *****************************/ static int usage(const char* exeName) { - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] [input] [output]\n", exeName); + DISPLAY( "Usage : \n"); + DISPLAY( " %s [arg] [input] [output] \n", exeName); DISPLAY( "\n"); - DISPLAY( "input : a filename\n"); + DISPLAY( "input : a filename \n"); DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); - DISPLAY( "Arguments :\n"); + DISPLAY( "Arguments : \n"); DISPLAY( " -1 : Fast compression (default) \n"); DISPLAY( " -9 : High compression \n"); DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION); - DISPLAY( " -z : force compression\n"); + DISPLAY( " -z : force compression \n"); DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( " -k : preserve source files(s) (default) \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); - DISPLAY( " -h/-H : display help/long help and exit\n"); + DISPLAY( " -h/-H : display help/long help and exit \n"); return 0; } @@ -151,33 +132,33 @@ static int usage_advanced(const char* exeName) usage(exeName); DISPLAY( "\n"); DISPLAY( "Advanced arguments :\n"); - DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( " -v : verbose mode\n"); + DISPLAY( " -V : display Version number and exit \n"); + DISPLAY( " -v : verbose mode \n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -t : test compressed file integrity\n"); DISPLAY( " -m : multiple input files (implies automatic output filenames)\n"); #ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories (sets also -m)\n"); + DISPLAY( " -r : operate recursively on directories (sets also -m) \n"); #endif DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n"); - DISPLAY( " -B# : Block size [4-7] (default : 7)\n"); - DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); + DISPLAY( " -B# : Block size [4-7] (default : 7) \n"); + DISPLAY( " -BD : Block dependency (improve compression ratio) \n"); /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */ - DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n"); + DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n"); DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); - DISPLAY( "Benchmark arguments :\n"); + DISPLAY( "Benchmark arguments : \n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); - DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); - DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+]\n"); - DISPLAY( " or predefined block size [4-7] (default: 7)\n"); + DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n"); + DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+] \n"); + DISPLAY( " or predefined block size [4-7] (default: 7) \n"); #if defined(ENABLE_LZ4C_LEGACY_OPTIONS) - DISPLAY( "Legacy arguments :\n"); - DISPLAY( " -c0 : fast compression\n"); - DISPLAY( " -c1 : high compression\n"); - DISPLAY( " -hc : high compression\n"); + DISPLAY( "Legacy arguments : \n"); + DISPLAY( " -c0 : fast compression \n"); + DISPLAY( " -c1 : high compression \n"); + DISPLAY( " -hc : high compression \n"); DISPLAY( " -y : overwrite output without prompting \n"); #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */ EXTENDED_HELP; @@ -207,7 +188,7 @@ static int usage_longhelp(const char* exeName) DISPLAY( "Compression levels : \n"); DISPLAY( "---------------------\n"); DISPLAY( "-0 ... -2 => Fast compression, all identicals\n"); - DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_MAX_CLEVEL); + DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX); DISPLAY( "\n"); DISPLAY( "stdin, stdout and the console : \n"); DISPLAY( "--------------------------------\n"); @@ -259,6 +240,13 @@ static void waitEnter(void) (void)getchar(); } +static const char* lastNameFromPath(const char* path) +{ + const char* name = strrchr(path, '/'); + if (name==NULL) name = strrchr(path, '\\'); /* windows */ + if (name==NULL) return path; + return name+1; +} /*! readU32FromChar() : @return : unsigned integer value reach from input in `char` format @@ -293,7 +281,7 @@ int main(int argc, const char** argv) const char nullOutput[] = NULL_OUTPUT; const char extension[] = LZ4_EXTENSION; size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); - const char* const exeName = argv[0]; + const char* const exeName = lastNameFromPath(argv[0]); #ifdef UTIL_HAS_CREATEFILELIST const char** extendedFileList = NULL; char* fileNamesBuf = NULL; @@ -305,12 +293,14 @@ int main(int argc, const char** argv) DISPLAY("Allocation error : not enough memory \n"); return 1; } + inFileNames[0] = stdinmark; LZ4IO_setOverwrite(0); /* lz4cat predefined behavior */ if (!strcmp(exeName, LZ4CAT)) { mode = om_decompress; LZ4IO_setOverwrite(1); + LZ4IO_setRemoveSrcFile(0); forceStdout=1; output_filename=stdoutmark; displayLevel=1; @@ -459,7 +449,7 @@ int main(int argc, const char** argv) break; #ifdef UTIL_HAS_CREATEFILELIST - /* recursive */ + /* recursive */ case 'r': recursive=1; /* without break */ #endif /* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */ @@ -473,7 +463,7 @@ int main(int argc, const char** argv) iters = readU32FromChar(&argument); argument--; BMK_setNotificationLevel(displayLevel); - BMK_SetNbSeconds(iters); + BMK_SetNbSeconds(iters); /* notification if displayLevel >= 3 */ } break; @@ -508,6 +498,18 @@ int main(int argc, const char** argv) } DISPLAYLEVEL(3, WELCOME_MESSAGE); +#ifdef _POSIX_C_SOURCE + DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE); +#endif +#ifdef _POSIX_VERSION + DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION); +#endif +#ifdef PLATFORM_POSIX_VERSION + DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); +#endif +#ifdef _FILE_OFFSET_BITS + DISPLAYLEVEL(4, "_FILE_OFFSET_BITS defined: %ldL\n", (long) _FILE_OFFSET_BITS); +#endif if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %i KB\n", (U32)(blockSize>>10)); if (multiple_inputs) { @@ -546,9 +548,12 @@ int main(int argc, const char** argv) DISPLAYLEVEL(1, "refusing to read from a console\n"); exit(1); } + /* if input==stdin and no output defined, stdout becomes default output */ + if (!strcmp(input_filename, stdinmark) && !output_filename) + output_filename = stdoutmark; /* No output filename ==> try to select one automatically (when possible) */ - while (!output_filename) { + while ((!output_filename) && (multiple_inputs==0)) { if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ size_t const inSize = strlen(input_filename); @@ -584,17 +589,18 @@ int main(int argc, const char** argv) } /* Check if output is defined as console; trigger an error in this case */ + if (!output_filename) output_filename = "*\\dummy^!//"; if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) { DISPLAYLEVEL(1, "refusing to write to console without -c\n"); exit(1); } - /* Downgrade notification level in stdout and multiple file mode */ if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; /* IO Stream/File */ LZ4IO_setNotificationLevel(displayLevel); + if (ifnIdx == 0) multiple_inputs = 0; if (mode == om_decompress) { if (multiple_inputs) operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION); diff --git a/programs/lz4io.c b/programs/lz4io.c index bc2ba95..640c76d 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -30,24 +30,26 @@ - The license of this source file is GPLv2. */ -/************************************** -* Compiler Options -**************************************/ -#define _LARGE_FILES /* Large file support on 32-bits AIX */ -#define _FILE_OFFSET_BITS 64 /* off_t width */ -#define _LARGEFILE_SOURCE +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ #endif + /***************************** * Includes *****************************/ -#include "util.h" /* Compiler options, UTIL_getFileStat */ +#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE, PLATFORM_POSIX_VERSION, __64BIT__ */ +#include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ #include <stdio.h> /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include <stdlib.h> /* malloc, free */ -#include <string.h> /* strcmp, strlen */ +#include <string.h> /* strerror, strcmp, strlen */ #include <time.h> /* clock */ #include <sys/types.h> /* stat64 */ #include <sys/stat.h> /* stat64 */ @@ -57,30 +59,14 @@ #include "lz4frame.h" -/****************************** -* OS-specific Includes -******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) -# include <fcntl.h> /* _O_BINARY */ -# include <io.h> /* _setmode, _fileno, _get_osfhandle */ -# if !defined(__DJGPP__) -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } -# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include <winioctl.h> /* FSCTL_SET_SPARSE */ -# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } -# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ -# define fseek _fseeki64 -# endif -# else -# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) -# define SET_SPARSE_FILE_MODE(file) -# endif -#else -# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__)) -# define fseek fseeko -# endif -# define SET_BINARY_MODE(file) -# define SET_SPARSE_FILE_MODE(file) +/* ************************************** +* Compiler Options +****************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ +# define fseek _fseeki64 +#endif +#if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define fseek fseeko #endif @@ -591,12 +577,15 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, char* dstFileName = (char*)malloc(FNSPACE); size_t ofnSize = FNSPACE; const size_t suffixSize = strlen(suffix); - cRess_t const ress = LZ4IO_createCResources(); + cRess_t ress; + + if (dstFileName == NULL) return ifntSize; /* not enough memory */ + ress = LZ4IO_createCResources(); /* loop on each file */ for (i=0; i<ifntSize; i++) { size_t const ifnSize = strlen(inFileNamesTable[i]); - if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); } + if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); if (dstFileName==NULL) { LZ4IO_freeCResources(ress); return ifntSize; } } strcpy(dstFileName, inFileNamesTable[i]); strcat(dstFileName, suffix); @@ -660,8 +649,10 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer storedSkips += (unsigned)(nb0T * sizeT); if (nb0T != seg0SizeT) { /* not all 0s */ - int const seekResult = fseek(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); + errno = 0; + { int const seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno)); + } storedSkips = 0; seg0SizeT -= nb0T; ptrT += nb0T; @@ -1022,7 +1013,7 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz size_t const suffixSize = strlen(suffix); dRess_t ress = LZ4IO_createDResources(); - if (outFileName==NULL) exit(1); /* not enough memory */ + if (outFileName==NULL) return ifntSize; /* not enough memory */ ress.dstFile = LZ4IO_openDstFile(stdoutmark); for (i=0; i<ifntSize; i++) { @@ -1032,7 +1023,7 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark); continue; } - if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) exit(1); } + if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) return ifntSize; } if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) { DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); skippedFiles++; diff --git a/programs/platform.h b/programs/platform.h new file mode 100644 index 0000000..f1040c0 --- /dev/null +++ b/programs/platform.h @@ -0,0 +1,143 @@ +/* + platform.h - compiler and OS detection + Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#ifndef PLATFORM_H_MODULE +#define PLATFORM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + + +/* ************************************** +* Compiler Options +****************************************/ +#if defined(_MSC_VER) +# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */ +# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ +# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ +# endif +#endif + + +/* ************************************** +* Detect 64-bit OS +* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros +****************************************/ +#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ + || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ + || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ + || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ + || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ + || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ + || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ +# if !defined(__64BIT__) +# define __64BIT__ 1 +# endif +#endif + + +/* ********************************************************* +* Turn on Large Files support (>4GB) for 32-bit Linux/Unix +***********************************************************/ +#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ +# if !defined(_FILE_OFFSET_BITS) +# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ +# endif +# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ +# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ +# endif +# if defined(_AIX) || defined(__hpux) +# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ +# endif +#endif + + +/* ************************************************************ +* Detect POSIX version +* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows +* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX +* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION +***************************************************************/ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ + || defined(__midipix__) || defined(__VMS)) +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \ + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ +# define PLATFORM_POSIX_VERSION 200112L +# else +# if defined(__linux__) || defined(__linux) +# define _POSIX_C_SOURCE 200112L /* use feature test macro */ +# endif +# include <unistd.h> /* declares _POSIX_VERSION */ +# if defined(_POSIX_VERSION) /* POSIX compliant */ +# define PLATFORM_POSIX_VERSION _POSIX_VERSION +# else +# define PLATFORM_POSIX_VERSION 0 +# endif +# endif +#endif +#if !defined(PLATFORM_POSIX_VERSION) +# define PLATFORM_POSIX_VERSION -1 +#endif + + +/*-********************************************* +* Detect if isatty() and fileno() are available +************************************************/ +#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) +# include <unistd.h> /* isatty */ +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include <io.h> /* _isatty */ +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# define IS_CONSOLE(stdStream) 0 +#endif + + +/****************************** +* OS-specific Includes +******************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) +# include <fcntl.h> /* _O_BINARY */ +# include <io.h> /* _setmode, _fileno, _get_osfhandle */ +# if !defined(__DJGPP__) +# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include <winioctl.h> /* FSCTL_SET_SPARSE */ +# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } +# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } +# else +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +# define SET_SPARSE_FILE_MODE(file) +# endif +#else +# define SET_BINARY_MODE(file) +# define SET_SPARSE_FILE_MODE(file) +#endif + + + +#if defined (__cplusplus) +} +#endif + +#endif /* PLATFORM_H_MODULE */ diff --git a/programs/util.h b/programs/util.h index 1ad61bc..f3ff1b2 100644 --- a/programs/util.h +++ b/programs/util.h @@ -15,11 +15,8 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ + #ifndef UTIL_H_MODULE #define UTIL_H_MODULE @@ -27,70 +24,49 @@ extern "C" { #endif -/* ************************************** -* Compiler Options -****************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ -#endif -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ - #define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ -#endif -#endif - - -/* Unix Large Files support (>4GB) */ -#if !defined(__LP64__) /* No point defining Large file for 64 bit */ -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ -# define _LARGEFILE_SOURCE /* fseeko, ftello */ -# elif !defined(_LARGEFILE64_SOURCE) -# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */ -# endif -#endif /*-**************************************** * Dependencies ******************************************/ -#include <stdlib.h> /* features.h with _POSIX_C_SOURCE, malloc */ -#include <stdio.h> /* fprintf */ -#include <string.h> /* strerr, strlen, memcpy */ -#include <stddef.h> /* ptrdiff_t */ -#include <sys/types.h> /* stat, utime */ -#include <sys/stat.h> /* stat */ +#include "platform.h" /* PLATFORM_POSIX_VERSION */ +#include <stdlib.h> /* malloc */ +#include <stddef.h> /* size_t, ptrdiff_t */ +#include <stdio.h> /* fprintf */ +#include <sys/types.h> /* stat, utime */ +#include <sys/stat.h> /* stat */ #if defined(_MSC_VER) - #include <sys/utime.h> /* utime */ - #include <io.h> /* _chmod */ +# include <sys/utime.h> /* utime */ +# include <io.h> /* _chmod */ #else - #include <unistd.h> /* chown, stat */ - #include <utime.h> /* utime */ +# include <unistd.h> /* chown, stat */ +# include <utime.h> /* utime */ #endif -#include <time.h> /* time */ +#include <time.h> /* time */ #include <errno.h> -/* ************************************* -* Constants -***************************************/ -#define LIST_SIZE_INCREASE (8*1024) - -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(__GNUC__) -# define UTIL_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define UTIL_STATIC static inline -#elif defined(_MSC_VER) -# define UTIL_STATIC static __inline +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; #else -# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; #endif @@ -102,7 +78,7 @@ extern "C" { # define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleepMilli(milli) Sleep(milli) -#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) +#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ # include <unistd.h> # include <sys/resource.h> /* setpriority */ # include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ @@ -112,7 +88,7 @@ extern "C" { # define SET_HIGH_PRIORITY /* disabled */ # endif # define UTIL_sleep(s) sleep(s) -# if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L) +# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ # define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } # else # define UTIL_sleepMilli(milli) /* disabled */ @@ -124,27 +100,27 @@ extern "C" { #endif -/*-************************************************************** -* Basic Types -*****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; +/* ************************************* +* Constants +***************************************/ +#define LIST_SIZE_INCREASE (8*1024) + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(__INTEL_COMPILER) +# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */ +#endif +#if defined(__GNUC__) +# define UTIL_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define UTIL_STATIC static inline +#elif defined(_MSC_VER) +# define UTIL_STATIC static __inline #else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif +# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif /*-**************************************** @@ -307,7 +283,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ { char* path; int dirLength, fnameLength, pathLength, nbFiles = 0; - WIN32_FIND_DATA cFile; + WIN32_FIND_DATAA cFile; HANDLE hFile; dirLength = (int)strlen(dirName); @@ -319,7 +295,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ path[dirLength+1] = '*'; path[dirLength+2] = 0; - hFile=FindFirstFile(path, &cFile); + hFile=FindFirstFileA(path, &cFile); if (hFile == INVALID_HANDLE_VALUE) { fprintf(stderr, "Cannot open directory '%s'\n", dirName); return 0; @@ -356,16 +332,16 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ } } free(path); - } while (FindNextFile(hFile, &cFile)); + } while (FindNextFileA(hFile, &cFile)); FindClose(hFile); return nbFiles; } -#elif (defined(__APPLE__) && defined(__MACH__)) || \ - ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */ +#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # define UTIL_HAS_CREATEFILELIST # include <dirent.h> /* opendir, readdir */ +# include <string.h> /* strerror, memcpy */ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { @@ -427,7 +403,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { (void)bufStart; (void)bufEnd; (void)pos; - fprintf(stderr, "Directory %s ignored (lz4 compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); + fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); return 0; } @@ -443,16 +419,15 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i { size_t pos; unsigned i, nbFiles; - char *bufend, *buf; + char* buf = (char*)malloc(LIST_SIZE_INCREASE); + char* bufend = buf + LIST_SIZE_INCREASE; const char** fileTable; - buf = (char*)malloc(LIST_SIZE_INCREASE); if (!buf) return NULL; - bufend = buf + LIST_SIZE_INCREASE; for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) { if (!UTIL_isDirectory(inputNames[i])) { - size_t len = strlen(inputNames[i]); + size_t const len = strlen(inputNames[i]); if (buf + pos + len >= bufend) { ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; buf = (char*)UTIL_realloc(buf, newListSize); @@ -474,8 +449,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); if (!fileTable) { free(buf); return NULL; } - for (i=0, pos=0; i<nbFiles; i++) - { + for (i=0, pos=0; i<nbFiles; i++) { fileTable[i] = buf + pos; pos += strlen(fileTable[i]) + 1; } diff --git a/tests/Makefile b/tests/Makefile index e58d449..97fa782 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -38,12 +38,12 @@ VOID := /dev/null TESTDIR := versionsTest PYTHON ?= python3 -CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make +CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 CFLAGS += $(MOREFLAGS) -CPPFLAGS:= -I$(LIBDIR) -DXXH_NAMESPACE=LZ4_ +CPPFLAGS:= -I$(LIBDIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_ FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) @@ -120,15 +120,21 @@ versionsTest: #------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#make test is validated only for Linux, OSX, kFreeBSD, FreeBSD, Hurd and +#Solaris targets +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) MD5:=md5sum ifneq (,$(filter $(shell uname), Darwin )) MD5:=md5 -r endif +DIFF:=diff +ifneq (,$(filter $(shell uname),SunOS)) +DIFF:=gdiff +endif + + test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer test32: CFLAGS+=-m32 @@ -138,18 +144,18 @@ test-lz4-sparse: lz4 datagen @echo "\n ---- test sparse file support ----" ./datagen -g5M -P100 > tmpSrc $(LZ4) -B4D tmpSrc | $(LZ4) -dv --sparse > tmpB4 - diff -s tmpSrc tmpB4 + $(DIFF) -s tmpSrc tmpB4 $(LZ4) -B5D tmpSrc | $(LZ4) -dv --sparse > tmpB5 - diff -s tmpSrc tmpB5 + $(DIFF) -s tmpSrc tmpB5 $(LZ4) -B6D tmpSrc | $(LZ4) -dv --sparse > tmpB6 - diff -s tmpSrc tmpB6 + $(DIFF) -s tmpSrc tmpB6 $(LZ4) -B7D tmpSrc | $(LZ4) -dv --sparse > tmpB7 - diff -s tmpSrc tmpB7 + $(DIFF) -s tmpSrc tmpB7 $(LZ4) tmpSrc | $(LZ4) -dv --no-sparse > tmpNoSparse - diff -s tmpSrc tmpNoSparse + $(DIFF) -s tmpSrc tmpNoSparse ls -ls tmp* ./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmpOdd # Odd size file (to generate non-full last block) - ./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd + ./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmpOdd ls -ls tmpOdd @$(RM) tmp* @echo "\n Compatibility with Console :" @@ -163,7 +169,7 @@ test-lz4-sparse: lz4 datagen $(LZ4) -d -v tmpC tmpR $(LZ4) -d -v tmpC >> tmpR ls -ls tmp* - diff tmp2M tmpR + $(DIFF) tmp2M tmpR @$(RM) tmp* test-lz4-contentSize: lz4 datagen @@ -171,13 +177,13 @@ test-lz4-contentSize: lz4 datagen ./datagen -g15M > tmp $(LZ4) -v tmp | $(LZ4) -t $(LZ4) -v --content-size tmp | $(LZ4) -d > tmp2 - diff -s tmp tmp2 + $(DIFF) -s tmp tmp2 # test large size [2-4] GB - @./datagen -g3G -P100 | $(LZ4) --verbose | $(LZ4) --decompress --force --sparse - tmp + @./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmp @ls -ls tmp @./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmp2 @ls -ls tmp2 - diff -s tmp tmp2 + $(DIFF) -s tmp tmp2 @$(RM) tmp* test-lz4-frame-concatenation: lz4 datagen @@ -206,20 +212,44 @@ test-lz4-multiple: lz4 datagen $(LZ4) -f -m tmp1 notHere tmp2; echo $$? @$(RM) tmp* -test-lz4-basic: lz4 datagen +unlz4: + @$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)" + +lz4cat: + @$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)" + +test-lz4-basic: lz4 datagen unlz4 lz4cat @echo "\n ---- test lz4 basic compression/decompression ----" ./datagen -g0 | $(LZ4) -v | $(LZ4) -t ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t ./datagen -g20KB > tmpSrc - $(LZ4) < tmpSrc | $(LZ4) -d > tmpRes - diff -q tmpSrc tmpRes + $(LZ4) < tmpSrc | $(LZ4) -d > tmpRes + $(DIFF) -q tmpSrc tmpRes $(LZ4) --no-frame-crc < tmpSrc | $(LZ4) -d > tmpRes - diff -q tmpSrc tmpRes + $(DIFF) -q tmpSrc tmpRes ./datagen | $(LZ4) | $(LZ4) -t ./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t ./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t + @echo "hello world" > tmp + $(LZ4) --rm -f tmp + ls -ls tmp && false || true # must fail (--rm) + ls -ls tmp.lz4 + $(PRGDIR)/lz4cat tmp.lz4 # must display hello world + ls -ls tmp.lz4 + $(PRGDIR)/unlz4 --rm tmp.lz4 + ls -ls tmp + ls -ls tmp.lz4 && false || true # must fail (--rm) + ls -ls tmp.lz4.lz4 && false || true # must fail (unlz4) + $(PRGDIR)/lz4cat tmp # pass-through mode + ls -ls tmp + ls -ls tmp.lz4 && false || true # must fail (lz4cat) + $(LZ4) tmp # creates tmp.lz4 + $(PRGDIR)/lz4cat < tmp.lz4 > tmp3 # checks lz4cat works with stdin (#285) + $(DIFF) -q tmp tmp3 + $(PRGDIR)/lz4cat < tmp > tmp2 # checks lz4cat works with stdin (#285) + $(DIFF) -q tmp tmp2 @$(RM) tmp* test-lz4-hugefile: lz4 datagen @@ -247,7 +277,20 @@ test-lz4-testmode: lz4 datagen $(LZ4) -fm file1-dne file2-dne && false || true $(LZ4) -fm file1-dne file2-dne && false || true -test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \ +test-lz4-opt-parser: lz4 datagen + @echo "\n ---- test opt-parser ----" + ./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t + ./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t + ./datagen -g256K | $(LZ4) -12B4D | $(LZ4) -t + ./datagen -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t + ./datagen -g1M | $(LZ4) -12B5 | $(LZ4) -t + ./datagen -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t + ./datagen -g4M | $(LZ4) -11vq | $(LZ4) -qt + ./datagen -g8M | $(LZ4) -11B4 | $(LZ4) -t + ./datagen -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t + ./datagen -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t + +test-lz4: lz4 datagen test-lz4-opt-parser test-lz4-basic test-lz4-multiple test-lz4-sparse \ test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \ test-lz4-hugefile diff --git a/tests/datagencli.c b/tests/datagencli.c index 0b78d37..c985197 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -27,31 +27,13 @@ /************************************** * Includes **************************************/ +#include "util.h" /* U32 */ #include <stdio.h> /* fprintf, stderr */ #include "datagen.h" /* RDG_generate */ #include "lz4.h" /* LZ4_VERSION_STRING */ /************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** * Constants **************************************/ #define KB *(1 <<10) diff --git a/tests/frametest.c b/tests/frametest.c index a99728f..e2e0f86 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -35,6 +35,7 @@ /*-************************************ * Includes **************************************/ +#include "util.h" /* U32 */ #include <stdlib.h> /* malloc, free */ #include <stdio.h> /* fprintf */ #include <string.h> /* strcmp */ @@ -45,25 +46,6 @@ #include "xxhash.h" /* XXH64 */ -/*-************************************ -* Basic Types -**************************************/ -#if !defined(__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - - /* unoptimized version; solves endianess & alignment issues */ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) { @@ -110,7 +92,7 @@ static clock_t g_clockTime = 0; *****************************************/ static U32 no_prompt = 0; static U32 displayLevel = 2; -static U32 pause = 0; +static U32 use_pause = 0; /*-******************************************************* @@ -204,6 +186,13 @@ int basicTests(U32 seed, double compressibility) FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + /* LZ4F_compressBound() : special case : srcSize == 0 */ + DISPLAYLEVEL(3, "LZ4F_compressBound(0) = "); + { size_t const cBound = LZ4F_compressBound(0, NULL); + if (cBound < 64 KB) goto _output_error; + DISPLAYLEVEL(3, " %u \n", (U32)cBound); + } + /* Special case : null-content frame */ testSize = 0; DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n"); @@ -723,7 +712,7 @@ _end: free(compressedBuffer); free(decodedBuffer); - if (pause) { + if (use_pause) { DISPLAY("press enter to finish \n"); (void)getchar(); } @@ -795,7 +784,7 @@ int main(int argc, const char** argv) break; case 'p': /* pause at the end */ argument++; - pause = 1; + use_pause = 1; break; case 'i': diff --git a/tests/fullbench.c b/tests/fullbench.c index 7d54492..f489392 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -23,20 +23,6 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -/************************************** -* Compiler Options -**************************************/ -/* Disable some Visual warning messages */ -#define _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ - -/* Unix Large Files support (>4GB) */ -#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions -# define _LARGEFILE_SOURCE -# define _FILE_OFFSET_BITS 64 -#elif ! defined(__LP64__) // No point defining Large file for 64 bit -# define _LARGEFILE64_SOURCE -#endif // S_ISREG & gettimeofday() are not supported by MSVC #if defined(_MSC_VER) || defined(_WIN32) @@ -47,8 +33,10 @@ /************************************** * Includes **************************************/ +#include "platform.h" /* _CRT_SECURE_NO_WARNINGS, Large Files support */ +#include "util.h" /* U32, UTIL_getFileSize */ #include <stdlib.h> /* malloc, free */ -#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <stdio.h> /* fprintf, fopen, ftello */ #include <sys/types.h> /* stat64 */ #include <sys/stat.h> /* stat64 */ #include <string.h> /* strcmp */ @@ -62,34 +50,6 @@ /************************************** -* Compiler Options -**************************************/ -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - - -/************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** * Constants **************************************/ #define PROGRAM_DESCRIPTION "LZ4 speed analyzer" @@ -194,21 +154,6 @@ static size_t BMK_findMaxMem(U64 requiredMem) } -static U64 BMK_GetFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - - /********************************************************* * Benchmark function *********************************************************/ @@ -398,7 +343,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } /* Memory size adjustments */ - inFileSize = BMK_GetFileSize(inFileName); + inFileSize = UTIL_getFileSize(inFileName); if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; } benchedSize = BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */ if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 06cfd40..b129c96 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -27,7 +27,6 @@ * Compiler options **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ # pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ @@ -37,6 +36,8 @@ /*-************************************ * Dependencies **************************************/ +#include "platform.h" /* _CRT_SECURE_NO_WARNINGS */ +#include "util.h" /* U32 */ #include <stdlib.h> #include <stdio.h> /* fgets, sscanf */ #include <string.h> /* strcmp */ @@ -49,19 +50,7 @@ /*-************************************ * Basic Types **************************************/ -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include <stdint.h> -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; +#if !defined(__cplusplus) && !(defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which doesn't need address overflow test) */ #endif @@ -314,6 +303,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; int const dictSize = MIN(dictSizeRand, blockStart); + int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1); char* const block = ((char*)CNBuffer) + blockStart; const char* dict = block - dictSize; int compressedSize, HCcompressedSize; @@ -360,18 +350,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression HC */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); HCcompressedSize = ret; /* Test compression HC using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); /* Test compression using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); + ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8); FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); /* Test compression */ @@ -477,12 +467,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, 9); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, 9); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); /* Test compression with missing bytes into output buffer => must fail */ @@ -502,7 +492,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, 9); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel); FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) } @@ -603,7 +593,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST; dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7); + LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); @@ -611,7 +601,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST; LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); + FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); FUZ_DISPLAYTEST; LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); @@ -666,7 +656,7 @@ _output_error: #define testCompressedSize (128 KB) #define ringBufferSize (8 KB) -static void FUZ_unitTests(void) +static void FUZ_unitTests(int compressionLevel) { const unsigned testNb = 0; const unsigned seed = 0; @@ -761,7 +751,7 @@ static void FUZ_unitTests(void) /* simple HC compression test */ crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); @@ -772,7 +762,7 @@ static void FUZ_unitTests(void) /* simple dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 64 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); @@ -786,7 +776,7 @@ static void FUZ_unitTests(void) { int result1, result2; int segSize = testCompressedSize / 2; crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); @@ -803,7 +793,7 @@ static void FUZ_unitTests(void) /* remote dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 32 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); @@ -824,7 +814,7 @@ static void FUZ_unitTests(void) int segSize = (FUZ_rand(&randState) & 8191); int segNb = 1; - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, dict, dictSize); XXH64_reset(&crcOrigState, 0); @@ -870,7 +860,7 @@ static void FUZ_unitTests(void) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNew, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeState, NULL, 0); while (iNext + messageSize < testCompressedSize) { @@ -912,7 +902,7 @@ static void FUZ_unitTests(void) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNew, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeState, NULL, 0); #define BSIZE1 65537 @@ -967,7 +957,7 @@ static void FUZ_unitTests(void) } } - printf("All unit tests completed successfully \n"); + printf("All unit tests completed successfully compressionLevel=%d \n", compressionLevel); return; _output_error: exit(1); @@ -1000,7 +990,7 @@ int main(int argc, const char** argv) int nbTests = NB_ATTEMPTS; int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; - int pause = 0; + int use_pause = 0; const char* programName = argv[0]; U32 duration = 0; @@ -1012,7 +1002,7 @@ int main(int argc, const char** argv) // Decode command (note : aggregated commands are allowed) if (argument[0]=='-') { - if (!strcmp(argument, "--no-prompt")) { pause=0; seedset=1; g_displayLevel=1; continue; } + if (!strcmp(argument, "--no-prompt")) { use_pause=0; seedset=1; g_displayLevel=1; continue; } argument++; while (*argument!=0) { @@ -1028,7 +1018,7 @@ int main(int argc, const char** argv) case 'p': /* pause at the end */ argument++; - pause=1; + use_pause=1; break; case 'i': @@ -1102,7 +1092,7 @@ int main(int argc, const char** argv) } } - printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); + printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_versionString()); if (!seedset) { time_t const t = time(NULL); @@ -1113,12 +1103,12 @@ int main(int argc, const char** argv) if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - if ((seedset==0) && (testNb==0)) FUZ_unitTests(); + if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); } if (nbTests<=0) nbTests=1; { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); - if (pause) { + if (use_pause) { DISPLAY("press enter ... \n"); (void)getchar(); } diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj index 363c5ae..76d12c9 100644 --- a/visual/VS2010/frametest/frametest.vcxproj +++ b/visual/VS2010/frametest/frametest.vcxproj @@ -66,20 +66,20 @@ <PropertyGroup Label="UserMacros" /> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj index 16a4f0d..c10552a 100644 --- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj +++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj @@ -66,20 +66,20 @@ <PropertyGroup Label="UserMacros" /> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj index 8d0a623..e2d95c9 100644 --- a/visual/VS2010/fullbench/fullbench.vcxproj +++ b/visual/VS2010/fullbench/fullbench.vcxproj @@ -66,20 +66,20 @@ <PropertyGroup Label="UserMacros" /> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj index 6672aaa..85d6c9b 100644 --- a/visual/VS2010/fuzzer/fuzzer.vcxproj +++ b/visual/VS2010/fuzzer/fuzzer.vcxproj @@ -66,20 +66,20 @@ <PropertyGroup Label="UserMacros" /> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <LinkIncremental>true</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <LinkIncremental>false</LinkIncremental> - <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> + <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath> <RunCodeAnalysis>true</RunCodeAnalysis> </PropertyGroup> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj index 295a94b..693e121 100644 --- a/visual/VS2010/lz4/lz4.vcxproj +++ b/visual/VS2010/lz4/lz4.vcxproj @@ -29,24 +29,24 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>true</UseDebugLibraries> - <CharacterSet>MultiByte</CharacterSet> + <CharacterSet>Unicode</CharacterSet> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>true</UseDebugLibraries> - <CharacterSet>MultiByte</CharacterSet> + <CharacterSet>Unicode</CharacterSet> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> <WholeProgramOptimization>true</WholeProgramOptimization> - <CharacterSet>MultiByte</CharacterSet> + <CharacterSet>Unicode</CharacterSet> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> <WholeProgramOptimization>true</WholeProgramOptimization> - <CharacterSet>MultiByte</CharacterSet> + <CharacterSet>Unicode</CharacterSet> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> |