diff options
-rw-r--r-- | .travis.yml | 106 | ||||
-rw-r--r-- | Makefile | 24 | ||||
-rw-r--r-- | NEWS | 7 | ||||
-rw-r--r-- | lib/lz4.c | 31 | ||||
-rw-r--r-- | lib/lz4hc.c | 12 | ||||
-rw-r--r-- | programs/lz4io.c | 21 | ||||
-rw-r--r-- | tests/Makefile | 4 |
7 files changed, 72 insertions, 133 deletions
diff --git a/.travis.yml b/.travis.yml index ef8732c..4d6bdc4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,57 +4,29 @@ matrix: include: # OS X Mavericks - os: osx - env: Ubu="OS X Mavericks" PARAMS='-C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang + env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang + # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - os: linux sudo: false - env: Ubu="12.04cont" PARAMS="install PREFIX=~/install_test_dir" COMPILER=cc + env: Ubu=12.04cont Cmd="make gpptest && make clean && make examples && make clean && make cmake && make clean && make travis-install" COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" PARAMS="examples" COMPILER=cc + env: Ubu=12.04cont Cmd='make test' COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" PARAMS="cmake" COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" PARAMS=clangtest COMPILER=clang - - os: linux - sudo: false - env: Ubu="12.04cont" PARAMS=gpptest COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-lz4' COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-lz4 MOREFLAGS="-std=c99"' COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-lz4c' COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-fullbench' COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-fuzzer' COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-frametest' COMPILER=cc + env: Ubu=12.04cont Cmd='make clangtest' COMPILER=clang + # 14.04 LTS Server Edition 64 bit - - env: Ubu="14.04" PARAMS=sanitize COMPILER=clang + - env: Ubu=14.04 Cmd='make staticAnalyze sanitize' COMPILER=clang dist: trusty sudo: required addons: apt: packages: - valgrind - - env: Ubu="14.04" PARAMS=staticAnalyze COMPILER=clang - dist: trusty - sudo: required - addons: - apt: - packages: - clang - - env: Ubu="14.04" PARAMS=armtest COMPILER=arm-linux-gnueabi-gcc + - env: Ubu=14.04 Cmd='make armtest' COMPILER=arm-linux-gnueabi-gcc dist: trusty sudo: required addons: @@ -63,36 +35,23 @@ matrix: - gcc-arm-linux-gnueabi - libc6-dev-armel-cross - linux-libc-dev-armel-cross - - env: Ubu="14.04" PARAMS=versionsTest COMPILER=cc + - env: Ubu=14.04 Cmd='make c_standards && make -C tests test-lz4 test-mem' COMPILER=cc dist: trusty sudo: required addons: apt: packages: - - python3 - - libc6-dev-i386 - - gcc-multilib - - env: Ubu="14.04" PARAMS='-C tests test-lz4' COMPILER=cc - dist: trusty - sudo: required - - env: Ubu="14.04" PARAMS='-C tests test-lz4c32' COMPILER=cc + - valgrind + - env: Ubu=14.04 Cmd='make -C tests test32 versionsTest' COMPILER=cc dist: trusty sudo: required addons: apt: packages: + - python3 - libc6-dev-i386 - gcc-multilib - - env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=gcc-5' COMPILER=gcc-5 - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-5 - - env: Ubu="14.04" PARAMS='-C tests test-lz4c32 CC=gcc-5' COMPILER=gcc-5 + - env: Ubu=14.04 Cmd='make -C tests test-lz4 test-lz4c32 CC=gcc-5 MOREFLAGS="-Werror"' COMPILER=gcc-5 dist: trusty sudo: required addons: @@ -104,7 +63,7 @@ matrix: - gcc-multilib - gcc-5 - gcc-5-multilib - - env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=gcc-6' COMPILER=gcc-6 + - env: Ubu=14.04 Cmd='make c_standards CC=gcc-6 && make -C tests test-lz4 CC=gcc-6 MOREFLAGS="-Werror"' COMPILER=gcc-6 dist: trusty sudo: required addons: @@ -113,7 +72,7 @@ matrix: - ubuntu-toolchain-r-test packages: - gcc-6 - - env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8 + - env: Ubu=14.04 Cmd='make -C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8 dist: trusty sudo: required addons: @@ -123,39 +82,8 @@ matrix: - llvm-toolchain-precise-3.8 packages: - clang-3.8 - - env: Ubu="14.04" PARAMS='-C tests test-fullbench32' COMPILER=cc - dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib - - env: Ubu="14.04" PARAMS='-C tests test-fuzzer32' COMPILER=cc - dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib - - env: Ubu="14.04" PARAMS='-C tests test-frametest32' COMPILER=cc - dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib - - env: Ubu="14.04" PARAMS='-C tests test-mem' COMPILER=cc - dist: trusty - sudo: required - addons: - apt: - packages: - - valgrind script: - - echo PARAMS=$PARAMS + - echo Cmd=$Cmd - $COMPILER -v - - sh -c "make $PARAMS" + - sh -c "$Cmd" @@ -85,7 +85,7 @@ uninstall: @$(MAKE) -C $(PRGDIR) $@ travis-install: - sudo $(MAKE) install + $(MAKE) install PREFIX=~/install_test_dir test: $(MAKE) -C $(TESTDIR) test @@ -96,19 +96,29 @@ cmake: gpptest: clean $(MAKE) all CC=g++ CFLAGS="-O3 -I../lib -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" +c_standards: clean + $(MAKE) all CFLAGS="-std=gnu90" + $(MAKE) clean + $(MAKE) all CFLAGS="-std=c99" + $(MAKE) clean + $(MAKE) all CFLAGS="-std=gnu99" + $(MAKE) clean + $(MAKE) all CFLAGS="-std=c11" + $(MAKE) clean + clangtest: clean - CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) all CC=clang + $(MAKE) all CC=clang CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" sanitize: clean - CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test CC=clang FUZZER_TIME="-T1mn" NB_LOOPS=-i1 + $(MAKE) test CC=clang FUZZER_TIME="-T1mn" NB_LOOPS=-i1 CFLAGS="-O3 -g -fsanitize=undefined" staticAnalyze: clean - CFLAGS=-g scan-build --status-bugs -v $(MAKE) all + scan-build --status-bugs -v $(MAKE) all CFLAGS=-g armtest: clean - CFLAGS="-O3 -Werror" $(MAKE) -C $(LZ4DIR) all CC=arm-linux-gnueabi-gcc - CFLAGS="-O3 -Werror" $(MAKE) -C $(PRGDIR) bins CC=arm-linux-gnueabi-gcc - CFLAGS="-O3 -Werror" $(MAKE) -C $(TESTDIR) bins CC=arm-linux-gnueabi-gcc + $(MAKE) -C $(LZ4DIR) all CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror" + $(MAKE) -C $(PRGDIR) bins CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror" + $(MAKE) -C $(TESTDIR) bins CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror" versionsTest: clean $(MAKE) -C $(TESTDIR) $@ @@ -2,12 +2,17 @@ v1.7.2 Changed : moved to versioning; package, cli and library have same version number Improved: Small decompression speed boost Improved: Small compression speed improvement on 64-bits systems -Improved: Performance on ARMv6 and ARMv7 +Improved: Small compression ratio and speed improvement on small files +Improved: Significant speed boost on ARMv6 and ARMv7 +New : cli : --rm command +New : cli : file attributes are preserved, by Przemyslaw Skibinki Added : Debianization, by Evgeniy Polyakov Makefile: Generates object files (*.o) for faster (re)compilation on low power systems +New : recursive mode in benchmark Fix : cli : crash on some invalid inputs Fix : cli : -t correctly validates lz4-compressed files, by Nick Terrell Fix : better ratio on 64-bits big-endian targets +Fix : cli : detects and reports fread() errors, thanks to Hiroshi Fujishima report #243 r131 New : Dos/DJGPP target, thanks to Louis Santillan (#114) @@ -412,7 +412,7 @@ int LZ4_sizeofState() { return LZ4_STREAMSIZE; } /*-****************************** * Compression functions ********************************/ -static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType) +static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { if (tableType == byU16) return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); @@ -420,7 +420,7 @@ static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType) return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } -static U32 LZ4_hashSequence64(U64 sequence, tableType_t const tableType) +static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { static const U64 prime5bytes = 889523592379ULL; static const U64 prime8bytes = 11400714785074694791ULL; @@ -431,15 +431,10 @@ static U32 LZ4_hashSequence64(U64 sequence, tableType_t const tableType) return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); } -static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType) +FORCE_INLINE U32 LZ4_hashPosition(const void* p, tableType_t const tableType) { - if (LZ4_64bits()) return LZ4_hashSequence64(sequence, tableType); - return LZ4_hashSequence((U32)sequence, tableType); -} - -static U32 LZ4_hashPosition(const void* p, tableType_t tableType) -{ - return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType); + if ((LZ4_64bits()) && (tableType == byU32)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + return LZ4_hash4(LZ4_read32(p), tableType); } static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) @@ -452,7 +447,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy } } -static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); @@ -465,7 +460,7 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tab { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); @@ -655,7 +650,7 @@ _next_match: _last_literals: /* Encode Last Literals */ - { const size_t lastRun = (size_t)(iend - anchor); + { size_t const lastRun = (size_t)(iend - anchor); if ( (outputLimited) && /* Check output buffer overflow */ ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) return 0; @@ -683,12 +678,12 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int if (maxOutputSize >= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } else { if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } @@ -701,10 +696,10 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else LZ4_stream_t ctx; - void* ctxPtr = &ctx; + void* const ctxPtr = &ctx; #endif - int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); + int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); #if (HEAPMODE) FREEMEM(ctxPtr); diff --git a/lib/lz4hc.c b/lib/lz4hc.c index f109622..edc2db0 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -332,11 +332,11 @@ FORCE_INLINE int LZ4HC_encodeSequence ( static int LZ4HC_compress_generic ( - void* ctxvoid, - const char* source, - char* dest, - int inputSize, - int maxOutputSize, + void* const ctxvoid, + const char* const source, + char* const dest, + int const inputSize, + int const maxOutputSize, int compressionLevel, limitedOutput_directive limit ) @@ -381,7 +381,7 @@ static int LZ4HC_compress_generic ( _Search2: if (ip+ml < mflimit) - ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts); + ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts); else ml2 = ml; if (ml2 == ml) { /* No better match */ diff --git a/programs/lz4io.c b/programs/lz4io.c index 02f4c2b..73957e1 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -466,20 +466,21 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* read first block */ readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile); + if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName); filesize += readSize; /* single-block file */ if (readSize < blockSize) { /* Compress in single pass */ size_t const cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs); - if (LZ4F_isError(cSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(cSize)); + if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize)); compressedfilesize = cSize; DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ /* Write Block */ { size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); - if (sizeCheck!=cSize) EXM_THROW(35, "Write error : cannot write compressed block"); + if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block"); } } else @@ -488,9 +489,9 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, { /* Write Archive Header */ size_t headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); - if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); + if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); } + if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } compressedfilesize += headerSize; /* Main Loop */ @@ -499,26 +500,26 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* Compress Block */ outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); - if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); + if (LZ4F_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize)); compressedfilesize += outSize; DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ { size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); - if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); } + if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); } /* Read next block */ readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); filesize += readSize; } - if (ferror(srcFile)) EXM_THROW(36, "Error reading %s ", srcFileName); + if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); /* End of Stream mark */ headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); - if (LZ4F_isError(headerSize)) EXM_THROW(37, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); + if (LZ4F_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(38, "Write error : cannot write end of stream"); } + if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); } compressedfilesize += headerSize; } @@ -532,7 +533,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, UTIL_setFileStat(dstFileName, &statbuf); } - if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(39, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ + if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); diff --git a/tests/Makefile b/tests/Makefile index 2da6408..d5aad60 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -122,9 +122,9 @@ versionsTest: #FreeBSD targets ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) -test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer test-mem +test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer -test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32 test-mem32 +test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32 test-all: test test32 |