Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev

7 files changed, 72 insertions, 133 deletions

diff --git a/.travis.yml b/.travis.yml
index ef8732c..4d6bdc4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,57 +4,29 @@ matrix:
   include:
     # OS X Mavericks
     - os: osx
-      env: Ubu="OS X Mavericks" PARAMS='-C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang
+      env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang
+    
     # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
     - os: linux
       sudo: false
-      env: Ubu="12.04cont" PARAMS="install PREFIX=~/install_test_dir" COMPILER=cc
+      env: Ubu=12.04cont Cmd="make gpptest && make clean && make examples && make clean && make cmake && make clean && make travis-install" COMPILER=cc
     - os: linux
       sudo: false
-      env: Ubu="12.04cont" PARAMS="examples" COMPILER=cc
+      env: Ubu=12.04cont Cmd='make test' COMPILER=cc
     - os: linux
       sudo: false
-      env: Ubu="12.04cont" PARAMS="cmake" COMPILER=cc
-    - os: linux
-      sudo: false
-      env: Ubu="12.04cont" PARAMS=clangtest COMPILER=clang
-    - os: linux
-      sudo: false
-      env: Ubu="12.04cont" PARAMS=gpptest COMPILER=cc
-    - os: linux
-      sudo: false
-      env: Ubu="12.04cont" PARAMS='-C tests test-lz4' COMPILER=cc
-    - os: linux
-      sudo: false
-      env: Ubu="12.04cont" PARAMS='-C tests test-lz4 MOREFLAGS="-std=c99"' COMPILER=cc
-    - os: linux
-      sudo: false
-      env: Ubu="12.04cont" PARAMS='-C tests test-lz4c' COMPILER=cc
-    - os: linux
-      sudo: false
-      env: Ubu="12.04cont" PARAMS='-C tests test-fullbench' COMPILER=cc
-    - os: linux
-      sudo: false
-      env: Ubu="12.04cont" PARAMS='-C tests test-fuzzer' COMPILER=cc
-    - os: linux
-      sudo: false
-      env: Ubu="12.04cont" PARAMS='-C tests test-frametest' COMPILER=cc
+      env: Ubu=12.04cont Cmd='make clangtest' COMPILER=clang
+    
     # 14.04 LTS Server Edition 64 bit
-    - env: Ubu="14.04" PARAMS=sanitize COMPILER=clang
+    - env: Ubu=14.04 Cmd='make staticAnalyze sanitize' COMPILER=clang
       dist: trusty
       sudo: required
       addons:
         apt:
           packages:
             - valgrind
-    - env: Ubu="14.04" PARAMS=staticAnalyze COMPILER=clang
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
             - clang
-    - env: Ubu="14.04" PARAMS=armtest COMPILER=arm-linux-gnueabi-gcc
+    - env: Ubu=14.04 Cmd='make armtest' COMPILER=arm-linux-gnueabi-gcc
       dist: trusty
       sudo: required
       addons:
@@ -63,36 +35,23 @@ matrix:
             - gcc-arm-linux-gnueabi
             - libc6-dev-armel-cross
             - linux-libc-dev-armel-cross
-    - env: Ubu="14.04" PARAMS=versionsTest COMPILER=cc
+    - env: Ubu=14.04 Cmd='make c_standards && make -C tests test-lz4 test-mem' COMPILER=cc
       dist: trusty
       sudo: required
       addons:
         apt:
           packages:
-            - python3
-            - libc6-dev-i386
-            - gcc-multilib
-    - env: Ubu="14.04" PARAMS='-C tests test-lz4' COMPILER=cc
-      dist: trusty
-      sudo: required
-    - env: Ubu="14.04" PARAMS='-C tests test-lz4c32' COMPILER=cc
+            - valgrind
+    - env: Ubu=14.04 Cmd='make -C tests test32 versionsTest' COMPILER=cc
       dist: trusty
       sudo: required
       addons:
         apt:
           packages:
+            - python3
             - libc6-dev-i386
             - gcc-multilib
-    - env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=gcc-5' COMPILER=gcc-5
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - gcc-5
-    - env: Ubu="14.04" PARAMS='-C tests test-lz4c32 CC=gcc-5' COMPILER=gcc-5
+    - env: Ubu=14.04 Cmd='make -C tests test-lz4 test-lz4c32 CC=gcc-5 MOREFLAGS="-Werror"' COMPILER=gcc-5
       dist: trusty
       sudo: required
       addons:
@@ -104,7 +63,7 @@ matrix:
             - gcc-multilib
             - gcc-5
             - gcc-5-multilib
-    - env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=gcc-6' COMPILER=gcc-6
+    - env: Ubu=14.04 Cmd='make c_standards CC=gcc-6 && make -C tests test-lz4 CC=gcc-6 MOREFLAGS="-Werror"' COMPILER=gcc-6
       dist: trusty
       sudo: required
       addons:
@@ -113,7 +72,7 @@ matrix:
             - ubuntu-toolchain-r-test
           packages:
             - gcc-6
-    - env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8
+    - env: Ubu=14.04 Cmd='make -C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8
       dist: trusty
       sudo: required
       addons:
@@ -123,39 +82,8 @@ matrix:
             - llvm-toolchain-precise-3.8
           packages:
             - clang-3.8
-    - env: Ubu="14.04" PARAMS='-C tests test-fullbench32' COMPILER=cc
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - libc6-dev-i386
-            - gcc-multilib
-    - env: Ubu="14.04" PARAMS='-C tests test-fuzzer32' COMPILER=cc
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - libc6-dev-i386
-            - gcc-multilib
-    - env: Ubu="14.04" PARAMS='-C tests test-frametest32' COMPILER=cc
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - libc6-dev-i386
-            - gcc-multilib
-    - env: Ubu="14.04" PARAMS='-C tests test-mem' COMPILER=cc
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - valgrind
 
 script:
-  - echo PARAMS=$PARAMS
+  - echo Cmd=$Cmd
   - $COMPILER -v
-  - sh -c "make $PARAMS"
+  - sh -c "$Cmd"
diff --git a/Makefile b/Makefile
index e98aafd..aeb2f4b 100644
--- a/Makefile
+++ b/Makefile
@@ -85,7 +85,7 @@ uninstall:
 	@$(MAKE) -C $(PRGDIR) $@
 
 travis-install:
-	sudo $(MAKE) install
+	$(MAKE) install PREFIX=~/install_test_dir
 
 test:
 	$(MAKE) -C $(TESTDIR) test
@@ -96,19 +96,29 @@ cmake:
 gpptest: clean
 	$(MAKE) all CC=g++ CFLAGS="-O3 -I../lib -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
+c_standards: clean
+	$(MAKE) all CFLAGS="-std=gnu90" 
+	$(MAKE) clean
+	$(MAKE) all CFLAGS="-std=c99" 
+	$(MAKE) clean
+	$(MAKE) all CFLAGS="-std=gnu99" 
+	$(MAKE) clean
+	$(MAKE) all CFLAGS="-std=c11" 
+	$(MAKE) clean
+
 clangtest: clean
-	CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) all CC=clang
+	$(MAKE) all CC=clang CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" 
 
 sanitize: clean
-	CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test CC=clang FUZZER_TIME="-T1mn" NB_LOOPS=-i1
+	$(MAKE) test CC=clang FUZZER_TIME="-T1mn" NB_LOOPS=-i1 CFLAGS="-O3 -g -fsanitize=undefined" 
 
 staticAnalyze: clean
-	CFLAGS=-g scan-build --status-bugs -v $(MAKE) all
+	scan-build --status-bugs -v $(MAKE) all CFLAGS=-g 
 
 armtest: clean
-	CFLAGS="-O3 -Werror" $(MAKE) -C $(LZ4DIR) all CC=arm-linux-gnueabi-gcc
-	CFLAGS="-O3 -Werror" $(MAKE) -C $(PRGDIR) bins CC=arm-linux-gnueabi-gcc
-	CFLAGS="-O3 -Werror" $(MAKE) -C $(TESTDIR) bins CC=arm-linux-gnueabi-gcc
+	$(MAKE) -C $(LZ4DIR) all CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror" 
+	$(MAKE) -C $(PRGDIR) bins CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror" 
+	$(MAKE) -C $(TESTDIR) bins CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror" 
 
 versionsTest: clean
 	$(MAKE) -C $(TESTDIR) $@
diff --git a/NEWS b/NEWS
index 7bfa3c5..c349991 100644
--- a/NEWS
+++ b/NEWS
@@ -2,12 +2,17 @@ v1.7.2
 Changed : moved to versioning; package, cli and library have same version number
 Improved: Small decompression speed boost
 Improved: Small compression speed improvement on 64-bits systems
-Improved: Performance on ARMv6 and ARMv7
+Improved: Small compression ratio and speed improvement on small files
+Improved: Significant speed boost on ARMv6 and ARMv7
+New : cli : --rm command
+New : cli : file attributes are preserved, by Przemyslaw Skibinki
 Added : Debianization, by Evgeniy Polyakov
 Makefile: Generates object files (*.o) for faster (re)compilation on low power systems
+New : recursive mode in benchmark
 Fix : cli : crash on some invalid inputs
 Fix : cli : -t correctly validates lz4-compressed files, by Nick Terrell
 Fix : better ratio on 64-bits big-endian targets
+Fix : cli : detects and reports fread() errors, thanks to Hiroshi Fujishima report #243
 
 r131
 New    : Dos/DJGPP target, thanks to Louis Santillan (#114)
diff --git a/lib/lz4.c b/lib/lz4.c
index 740ae64..53b503c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -412,7 +412,7 @@ int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
 /*-******************************
 *  Compression functions
 ********************************/
-static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType)
+static U32 LZ4_hash4(U32 sequence, tableType_t const tableType)
 {
     if (tableType == byU16)
         return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
@@ -420,7 +420,7 @@ static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType)
         return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
 }
 
-static U32 LZ4_hashSequence64(U64 sequence, tableType_t const tableType)
+static U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
 {
     static const U64 prime5bytes = 889523592379ULL;
     static const U64 prime8bytes = 11400714785074694791ULL;
@@ -431,15 +431,10 @@ static U32 LZ4_hashSequence64(U64 sequence, tableType_t const tableType)
         return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
 }
 
-static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType)
+FORCE_INLINE U32 LZ4_hashPosition(const void* p, tableType_t const tableType)
 {
-    if (LZ4_64bits()) return LZ4_hashSequence64(sequence, tableType);
-    return LZ4_hashSequence((U32)sequence, tableType);
-}
-
-static U32 LZ4_hashPosition(const void* p, tableType_t tableType)
-{
-    return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType);
+    if ((LZ4_64bits()) && (tableType == byU32)) return LZ4_hash5(LZ4_read_ARCH(p), tableType);
+    return LZ4_hash4(LZ4_read32(p), tableType);
 }
 
 static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
@@ -452,7 +447,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy
     }
 }
 
-static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     U32 const h = LZ4_hashPosition(p, tableType);
     LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
@@ -465,7 +460,7 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tab
     { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
 }
 
-static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     U32 const h = LZ4_hashPosition(p, tableType);
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
@@ -655,7 +650,7 @@ _next_match:
 
 _last_literals:
     /* Encode Last Literals */
-    {   const size_t lastRun = (size_t)(iend - anchor);
+    {   size_t const lastRun = (size_t)(iend - anchor);
         if ( (outputLimited) &&  /* Check output buffer overflow */
             ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) )
             return 0;
@@ -683,12 +678,12 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit)
-            return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16,                        noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(state, source, dest, inputSize,             0,    notLimited,                        byU16, noDict, noDictIssue, acceleration);
         else
-            return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(state, source, dest, inputSize,             0,    notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
     } else {
         if (inputSize < LZ4_64Klimit)
-            return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16,                        noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput,                        byU16, noDict, noDictIssue, acceleration);
         else
             return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
     }
@@ -701,10 +696,10 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
 #else
     LZ4_stream_t ctx;
-    void* ctxPtr = &ctx;
+    void* const ctxPtr = &ctx;
 #endif
 
-    int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
+    int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
 
 #if (HEAPMODE)
     FREEMEM(ctxPtr);
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f109622..edc2db0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -332,11 +332,11 @@ FORCE_INLINE int LZ4HC_encodeSequence (
 
 
 static int LZ4HC_compress_generic (
-    void* ctxvoid,
-    const char* source,
-    char* dest,
-    int inputSize,
-    int maxOutputSize,
+    void* const ctxvoid,
+    const char* const source,
+    char* const dest,
+    int const inputSize,
+    int const maxOutputSize,
     int compressionLevel,
     limitedOutput_directive limit
     )
@@ -381,7 +381,7 @@ static int LZ4HC_compress_generic (
 
 _Search2:
         if (ip+ml < mflimit)
-            ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts);
+            ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts);
         else ml2 = ml;
 
         if (ml2 == ml) { /* No better match */
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 02f4c2b..73957e1 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -466,20 +466,21 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
 
     /* read first block */
     readSize  = fread(srcBuffer, (size_t)1, blockSize, srcFile);
+    if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName);
     filesize += readSize;
 
     /* single-block file */
     if (readSize < blockSize) {
         /* Compress in single pass */
         size_t const cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs);
-        if (LZ4F_isError(cSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(cSize));
+        if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
         compressedfilesize = cSize;
         DISPLAYUPDATE(2, "\rRead : %u MB   ==> %.2f%%   ",
                       (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100);   /* avoid division by zero */
 
         /* Write Block */
         {   size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile);
-            if (sizeCheck!=cSize) EXM_THROW(35, "Write error : cannot write compressed block");
+            if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block");
     }   }
 
     else
@@ -488,9 +489,9 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
     {
         /* Write Archive Header */
         size_t headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs);
-        if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
+        if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
         { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
-          if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); }
+          if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); }
         compressedfilesize += headerSize;
 
         /* Main Loop */
@@ -499,26 +500,26 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
 
             /* Compress Block */
             outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
-            if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize));
+            if (LZ4F_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
             compressedfilesize += outSize;
             DISPLAYUPDATE(2, "\rRead : %u MB   ==> %.2f%%   ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
 
             /* Write Block */
             { size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile);
-              if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); }
+              if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); }
 
             /* Read next block */
             readSize  = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile);
             filesize += readSize;
         }
-        if (ferror(srcFile)) EXM_THROW(36, "Error reading %s ", srcFileName);
+        if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName);
 
         /* End of Stream mark */
         headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
-        if (LZ4F_isError(headerSize)) EXM_THROW(37, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
+        if (LZ4F_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
 
         { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
-          if (sizeCheck!=headerSize) EXM_THROW(38, "Write error : cannot write end of stream"); }
+          if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); }
         compressedfilesize += headerSize;
     }
 
@@ -532,7 +533,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
             UTIL_setFileStat(dstFileName, &statbuf);
     }
 
-    if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(39, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */
+    if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */
 
     /* Final Status */
     DISPLAYLEVEL(2, "\r%79s\r", "");
diff --git a/tests/Makefile b/tests/Makefile
index 2da6408..d5aad60 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -122,9 +122,9 @@ versionsTest:
 #FreeBSD targets
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD))
 
-test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer test-mem
+test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer
 
-test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32 test-mem32
+test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32
 
 test-all: test test32