Merge branch 'dev' into extraInput

38 files changed, 259 insertions, 178 deletions

diff --git a/.travis.yml b/.travis.yml
index 354cb63..6074f08 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -122,6 +122,21 @@ matrix:
         - make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static
         - make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static
 
+    - name: aarch64 real-hw tests
+      arch: arm64
+      script:
+        - make test
+
+    - name: PPC64LE real-hw tests
+      arch: ppc64le
+      script:
+        - make test
+
+    - name: IBM s390x real-hw tests
+      arch: s390x
+      script:
+        - make test
+
     - name: (Xenial) gcc-5 compilation
       dist: xenial
       install:
diff --git a/Makefile b/Makefile
index f25f951..ab695ea 100644
--- a/Makefile
+++ b/Makefile
@@ -98,7 +98,7 @@ travis-install:
 	$(MAKE) -j1 install DESTDIR=~/install_test_dir
 
 cmake:
-	@cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
+	@cd build/cmake; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
 
 endif
 
diff --git a/NEWS b/NEWS
index 44c45be..34ca486 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,7 @@
 v1.9.3
 perf: faster speed on Visual Studio, by @wolfpld
+build: contrib/cmake_unofficial/ moved to build/cmake/
+build: visual/* moved to build/
 
 
 v1.9.2
diff --git a/appveyor.yml b/appveyor.yml
index 056719a..57b6dda 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -82,20 +82,20 @@ build_script:
       ECHO *** &&
       ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /p:EnableWholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "build\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /p:EnableWholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "build\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "build\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      COPY visual\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe programs\
+      msbuild "build\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe programs\
     )
 
 test_script:
diff --git a/visual/.gitignore b/build/.gitignore
index 69e1111..69e1111 100644
--- a/visual/.gitignore
+++ b/build/.gitignore
diff --git a/visual/README.md b/build/README.md
index 216971f..d416aeb 100644
--- a/visual/README.md
+++ b/build/README.md
@@ -4,7 +4,9 @@ Projects for various integrated development environments (IDE)
 #### Included projects
 
 The following projects are included with the lz4 distribution:
+- `cmake` - CMake project
 - `VS2010` - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015)
+- `VS2017` - Visual Studio 2017 project
 
 
 #### How to compile lz4 with Visual Studio
@@ -12,25 +14,25 @@ The following projects are included with the lz4 distribution:
 1. Install Visual Studio e.g. VS 2015 Community Edition (it's free).
 2. Download the latest version of lz4 from https://github.com/lz4/lz4/releases
 3. Decompress ZIP archive.
-4. Go to decompressed directory then to `visual` then `VS2010` and open `lz4.sln`
+4. Go to decompressed directory then to `build` then `VS2010` and open `lz4.sln`
 5. Visual Studio will ask about converting VS2010 project to VS2015 and you should agree.
 6. Change `Debug` to `Release` and if you have 64-bit Windows change also `Win32` to `x64`.
 7. Press F7 on keyboard or select `BUILD` from the menu bar and choose `Build Solution`.
-8. If compilation will be fine a compiled executable will be in `visual\VS2010\bin\x64_Release\lz4.exe`
+8. If compilation will be fine a compiled executable will be in `build\VS2010\bin\x64_Release\lz4.exe`
 
 
 #### Projects available within lz4.sln
 
 The Visual Studio solution file `lz4.sln` contains many projects that will be compiled to the
-`visual\VS2010\bin\$(Platform)_$(Configuration)` directory. For example `lz4` set to `x64` and
-`Release` will be compiled to `visual\VS2010\bin\x64_Release\lz4.exe`. The solution file contains the
+`build\VS2010\bin\$(Platform)_$(Configuration)` directory. For example `lz4` set to `x64` and
+`Release` will be compiled to `build\VS2010\bin\x64_Release\lz4.exe`. The solution file contains the
 following projects:
 
 - `lz4` : Command Line Utility, supporting gzip-like arguments
 - `datagen` : Synthetic and parametrable data generator, for tests
 - `frametest` : Test tool that checks lz4frame integrity on target platform
 - `fullbench`  : Precisely measure speed for each lz4 inner functions
-- `fuzzer` : Test tool, to check lz4 integrity on target platform 
+- `fuzzer` : Test tool, to check lz4 integrity on target platform
 - `liblz4` : A static LZ4 library compiled to `liblz4_static.lib`
 - `liblz4-dll` : A dynamic LZ4 library (DLL) compiled to `liblz4.dll` with the import library `liblz4.lib`
 - `fullbench-dll` : The fullbench program compiled with the import library; the executable requires LZ4 DLL
@@ -39,8 +41,8 @@ following projects:
 #### Using LZ4 DLL with Microsoft Visual C++ project
 
 The header files `lib\lz4.h`, `lib\lz4hc.h`, `lib\lz4frame.h` and the import library
-`visual\VS2010\bin\$(Platform)_$(Configuration)\liblz4.lib` are required to compile a
-project using Visual C++.
+`build\VS2010\bin\$(Platform)_$(Configuration)\liblz4.lib` are required to
+compile a project using Visual C++.
 
 1. The path to header files should be added to `Additional Include Directories` that can
    be found in Project Properties of Visual Studio IDE in the `C/C++` Property Pages on the `General` page.
@@ -50,4 +52,4 @@ project using Visual C++.
    then the directory has to be added to `Linker\General\Additional Library Directories`.
 
 The compiled executable will require LZ4 DLL which is available at
-`visual\VS2010\bin\$(Platform)_$(Configuration)\liblz4.dll`.
+`build\VS2010\bin\$(Platform)_$(Configuration)\liblz4.dll`.
diff --git a/visual/VS2010/datagen/datagen.vcxproj b/build/VS2010/datagen/datagen.vcxproj
index e24f961..e24f961 100644
--- a/visual/VS2010/datagen/datagen.vcxproj
+++ b/build/VS2010/datagen/datagen.vcxproj
diff --git a/visual/VS2010/frametest/frametest.vcxproj b/build/VS2010/frametest/frametest.vcxproj
index 3196768..3196768 100644
--- a/visual/VS2010/frametest/frametest.vcxproj
+++ b/build/VS2010/frametest/frametest.vcxproj
diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
index 8f503f5..8f503f5 100644
--- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
+++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj
diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj
index aa67431..aa67431 100644
--- a/visual/VS2010/fullbench/fullbench.vcxproj
+++ b/build/VS2010/fullbench/fullbench.vcxproj
diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj
index 21cbf56..21cbf56 100644
--- a/visual/VS2010/fuzzer/fuzzer.vcxproj
+++ b/build/VS2010/fuzzer/fuzzer.vcxproj
diff --git a/visual/VS2017/liblz4-dll/liblz4-dll.rc b/build/VS2010/liblz4-dll/liblz4-dll.rc
index b1871fe..b1871fe 100644
--- a/visual/VS2017/liblz4-dll/liblz4-dll.rc
+++ b/build/VS2010/liblz4-dll/liblz4-dll.rc
diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj b/build/VS2010/liblz4-dll/liblz4-dll.vcxproj
index 56ec3b9..56ec3b9 100644
--- a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
+++ b/build/VS2010/liblz4-dll/liblz4-dll.vcxproj
diff --git a/visual/VS2010/liblz4/liblz4.vcxproj b/build/VS2010/liblz4/liblz4.vcxproj
index 61ea159..61ea159 100644
--- a/visual/VS2010/liblz4/liblz4.vcxproj
+++ b/build/VS2010/liblz4/liblz4.vcxproj
diff --git a/visual/VS2010/lz4.sln b/build/VS2010/lz4.sln
index 78f223b..78f223b 100644
--- a/visual/VS2010/lz4.sln
+++ b/build/VS2010/lz4.sln
diff --git a/visual/VS2017/lz4/lz4.rc b/build/VS2010/lz4/lz4.rc
index c593edf..c593edf 100644
--- a/visual/VS2017/lz4/lz4.rc
+++ b/build/VS2010/lz4/lz4.rc
diff --git a/visual/VS2010/lz4/lz4.vcxproj b/build/VS2010/lz4/lz4.vcxproj
index de7a714..de7a714 100644
--- a/visual/VS2010/lz4/lz4.vcxproj
+++ b/build/VS2010/lz4/lz4.vcxproj
diff --git a/visual/VS2017/datagen/datagen.vcxproj b/build/VS2017/datagen/datagen.vcxproj
index 30e159e..30e159e 100644
--- a/visual/VS2017/datagen/datagen.vcxproj
+++ b/build/VS2017/datagen/datagen.vcxproj
diff --git a/visual/VS2017/frametest/frametest.vcxproj b/build/VS2017/frametest/frametest.vcxproj
index a3a403d..a3a403d 100644
--- a/visual/VS2017/frametest/frametest.vcxproj
+++ b/build/VS2017/frametest/frametest.vcxproj
diff --git a/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj b/build/VS2017/fullbench-dll/fullbench-dll.vcxproj
index d54a8d7..d54a8d7 100644
--- a/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj
+++ b/build/VS2017/fullbench-dll/fullbench-dll.vcxproj
diff --git a/visual/VS2017/fullbench/fullbench.vcxproj b/build/VS2017/fullbench/fullbench.vcxproj
index 54c9743..54c9743 100644
--- a/visual/VS2017/fullbench/fullbench.vcxproj
+++ b/build/VS2017/fullbench/fullbench.vcxproj
diff --git a/visual/VS2017/fuzzer/fuzzer.vcxproj b/build/VS2017/fuzzer/fuzzer.vcxproj
index aa6fe42..aa6fe42 100644
--- a/visual/VS2017/fuzzer/fuzzer.vcxproj
+++ b/build/VS2017/fuzzer/fuzzer.vcxproj
diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.rc b/build/VS2017/liblz4-dll/liblz4-dll.rc
index b1871fe..b1871fe 100644
--- a/visual/VS2010/liblz4-dll/liblz4-dll.rc
+++ b/build/VS2017/liblz4-dll/liblz4-dll.rc
diff --git a/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj b/build/VS2017/liblz4-dll/liblz4-dll.vcxproj
index 8e7ee3b..8e7ee3b 100644
--- a/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj
+++ b/build/VS2017/liblz4-dll/liblz4-dll.vcxproj
diff --git a/visual/VS2017/liblz4/liblz4.vcxproj b/build/VS2017/liblz4/liblz4.vcxproj
index 948f7db..948f7db 100644
--- a/visual/VS2017/liblz4/liblz4.vcxproj
+++ b/build/VS2017/liblz4/liblz4.vcxproj
diff --git a/visual/VS2017/lz4.sln b/build/VS2017/lz4.sln
index 6a2779f..6a2779f 100644
--- a/visual/VS2017/lz4.sln
+++ b/build/VS2017/lz4.sln
diff --git a/visual/VS2010/lz4/lz4.rc b/build/VS2017/lz4/lz4.rc
index c593edf..c593edf 100644
--- a/visual/VS2010/lz4/lz4.rc
+++ b/build/VS2017/lz4/lz4.rc
diff --git a/visual/VS2017/lz4/lz4.vcxproj b/build/VS2017/lz4/lz4.vcxproj
index b4fed24..b4fed24 100644
--- a/visual/VS2017/lz4/lz4.vcxproj
+++ b/build/VS2017/lz4/lz4.vcxproj
diff --git a/contrib/cmake_unofficial/.gitignore b/build/cmake/.gitignore
index d39505d..d39505d 100644
--- a/contrib/cmake_unofficial/.gitignore
+++ b/build/cmake/.gitignore
diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/build/cmake/CMakeLists.txt
index 42d92ea..42d92ea 100644
--- a/contrib/cmake_unofficial/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
diff --git a/contrib/debian/rules b/contrib/debian/rules
index 748e68d..c897bc5 100755
--- a/contrib/debian/rules
+++ b/contrib/debian/rules
@@ -4,5 +4,4 @@ include /usr/share/cdbs/1/rules/debhelper.mk
 include /usr/share/cdbs/1/class/cmake.mk
 
 
-DEB_CMAKE_EXTRA_FLAGS := -DCMAKE_BUILD_TYPE=RelWithDebInfo ../cmake_unofficial
-
+DEB_CMAKE_EXTRA_FLAGS := -DCMAKE_BUILD_TYPE=RelWithDebInfo ../../build/cmake
diff --git a/contrib/gen_manual/gen_manual.cpp b/contrib/gen_manual/gen_manual.cpp
index bedef94..d5fe702 100644
--- a/contrib/gen_manual/gen_manual.cpp
+++ b/contrib/gen_manual/gen_manual.cpp
@@ -96,10 +96,9 @@ void print_line(stringstream &sout, string line)
     epos = line.find("*/");
     if (spos!=string::npos && epos!=string::npos) {
         sout << line.substr(0, spos);
-        sout << "</b>" << line.substr(spos) << "<b>" << endl;
+        sout << "</b>" << line.substr(spos) << "<b>" << '\n';
     } else {
-      //  fprintf(stderr, "lines=%s\n", line.c_str());
-        sout << line << endl;
+        sout << line << '\n';
     }
 }
 
diff --git a/lib/README.md b/lib/README.md
index 707d777..3653c81 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -46,11 +46,11 @@ and `LZ4F_PUBLISH_STATIC_FUNCTIONS`.
 
 #### Build macros
 
-The following build macro can be selected at compilation time :
+The following build macro can be selected to adjust source code behavior at compilation time :
 
-- `LZ4_FAST_DEC_LOOP` : this triggers the optimized decompression loop.
-  This loops works great on x86/x64 cpus, and is automatically enabled on this platform.
-  It's possible to enable or disable it manually, by passing `LZ4_FAST_DEC_LOOP=1` or `0` to the preprocessor.
+- `LZ4_FAST_DEC_LOOP` : this triggers a speed optimized decompression loop, more powerful on modern cpus.
+  This loop works great on x86, x64 and aarch64 cpus, and is automatically enabled for them.
+  It's also possible to enable or disable it manually, by passing `LZ4_FAST_DEC_LOOP=1` or `0` to the preprocessor.
   For example, with `gcc` : `-DLZ4_FAST_DEC_LOOP=1`,
   and with `make` : `CPPFLAGS+=-DLZ4_FAST_DEC_LOOP=1 make lz4`.
 
@@ -66,9 +66,17 @@ The following build macro can be selected at compilation time :
   Should this be a problem, it's generally possible to make the compiler ignore these warnings,
   for example with `-Wno-deprecated-declarations` on `gcc`,
   or `_CRT_SECURE_NO_WARNINGS` for Visual Studio.
-  Another method is to define `LZ4_DISABLE_DEPRECATE_WARNINGS`
+  Another project-specific method is to define `LZ4_DISABLE_DEPRECATE_WARNINGS`
   before including the LZ4 header files.
 
+- `LZ4_FORCE_SW_BITCOUNT` : by default, the compression algorithm tries to determine lengths
+  by using bitcount instructions, generally implemented as fast single instructions in many cpus.
+  In case the target cpus doesn't support it, or compiler intrinsic doesn't work, or feature bad performance,
+  it's possible to use an optimized software path instead.
+  This is achieved by setting this build macros .
+  In most cases, it's not expected to be necessary,
+  but it can be legitimately considered for less common platforms.
+
 
 #### Amalgamation
 
@@ -103,7 +111,7 @@ The compiled executable will require LZ4 DLL which is available at `dll\liblz4.d
 
 #### Miscellaneous
 
-Other files present in the directory are not source code. There are :
+Other files present in the directory are not source code. They are :
 
  - `LICENSE` : contains the BSD license text
  - `Makefile` : `make` script to compile and install lz4 library (static and dynamic)
diff --git a/lib/lz4.c b/lib/lz4.c
index 162a36a..06d24da 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -88,6 +88,7 @@
  * Define this parameter if your target system or compiler does not support hardware bit count
  */
 #if defined(_MSC_VER) && defined(_WIN32_WCE)   /* Visual Studio for WinCE doesn't support Hardware bit count */
+#  undef  LZ4_FORCE_SW_BITCOUNT  /* avoid double def */
 #  define LZ4_FORCE_SW_BITCOUNT
 #endif
 
@@ -487,75 +488,92 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si
 **************************************/
 static unsigned LZ4_NbCommonBytes (reg_t val)
 {
+    assert(val != 0);
     if (LZ4_isLittleEndian()) {
-        if (sizeof(val)==8) {
-#       if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
+        if (sizeof(val) == 8) {
+#       if defined(_MSC_VER) && (_MSC_VER >= 1800) && defined(_M_AMD64) && !defined(LZ4_FORCE_SW_BITCOUNT)
+            /* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */
+            return (unsigned)_tzcnt_u64(val) >> 3;
+#       elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
             unsigned long r = 0;
-            _BitScanForward64( &r, (U64)val );
-            return (int)(r>>3);
-#       elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
+            _BitScanForward64(&r, (U64)val);
+            return (unsigned)r >> 3;
+#       elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \
+                            ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \
+                                        !defined(LZ4_FORCE_SW_BITCOUNT)
             return (unsigned)__builtin_ctzll((U64)val) >> 3;
 #       else
-            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
-                                                     0, 3, 1, 3, 1, 4, 2, 7,
-                                                     0, 2, 3, 6, 1, 5, 3, 5,
-                                                     1, 3, 4, 4, 2, 5, 6, 7,
-                                                     7, 0, 1, 2, 3, 3, 4, 6,
-                                                     2, 6, 5, 5, 3, 4, 5, 6,
-                                                     7, 1, 2, 4, 6, 4, 4, 5,
-                                                     7, 2, 6, 5, 7, 6, 7, 7 };
-            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+            const U64 m = 0x0101010101010101ULL;
+            val ^= val - 1;
+            return (unsigned)(((U64)((val & (m - 1)) * m)) >> 56);
 #       endif
         } else /* 32 bits */ {
 #       if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
             unsigned long r;
-            _BitScanForward( &r, (U32)val );
-            return (int)(r>>3);
-#       elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
+            _BitScanForward(&r, (U32)val);
+            return (unsigned)r >> 3;
+#       elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \
+                            ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \
+                                        !defined(LZ4_FORCE_SW_BITCOUNT)
             return (unsigned)__builtin_ctz((U32)val) >> 3;
 #       else
-            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
-                                                     3, 2, 2, 1, 3, 2, 0, 1,
-                                                     3, 3, 1, 2, 2, 2, 2, 0,
-                                                     3, 1, 2, 0, 1, 0, 1, 1 };
-            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+            const U32 m = 0x01010101;
+            return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24;
 #       endif
         }
     } else   /* Big Endian CPU */ {
-        if (sizeof(val)==8) {   /* 64-bits */
-#       if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            unsigned long r = 0;
-            _BitScanReverse64( &r, val );
-            return (unsigned)(r>>3);
-#       elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
+        if (sizeof(val)==8) {
+#       if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \
+                            ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \
+                                        !defined(LZ4_FORCE_SW_BITCOUNT)
             return (unsigned)__builtin_clzll((U64)val) >> 3;
 #       else
+#if 1
+            /* this method is probably faster,
+             * but adds a 128 bytes lookup table */
+            static const unsigned char ctz7_tab[128] = {
+                7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+                4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+                5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+                4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+                6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+                4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+                5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+                4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+            };
+            U64 const mask = 0x0101010101010101ULL;
+            U64 const t = (((val >> 8) - mask) | val) & mask;
+            return ctz7_tab[(t * 0x0080402010080402ULL) >> 57];
+#else
+            /* this method doesn't consume memory space like the previous one,
+             * but it contains several branches,
+             * that may end up slowing execution */
             static const U32 by32 = sizeof(val)*4;  /* 32 on 64 bits (goal), 16 on 32 bits.
-                Just to avoid some static analyzer complaining about shift by 32 on 32-bits target.
-                Note that this code path is never triggered in 32-bits mode. */
+            Just to avoid some static analyzer complaining about shift by 32 on 32-bits target.
+            Note that this code path is never triggered in 32-bits mode. */
             unsigned r;
             if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; }
             if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
             r += (!val);
             return r;
+#endif
 #       endif
         } else /* 32 bits */ {
-#       if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            unsigned long r = 0;
-            _BitScanReverse( &r, (unsigned long)val );
-            return (unsigned)(r>>3);
-#       elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
+#       if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \
+                            ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \
+                                        !defined(LZ4_FORCE_SW_BITCOUNT)
             return (unsigned)__builtin_clz((U32)val) >> 3;
 #       else
-            unsigned r;
-            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
-            r += (!val);
-            return r;
+            val >>= 8;
+            val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) |
+              (val + 0x00FF0000)) >> 24;
+            return (unsigned)val ^ 3;
 #       endif
         }
     }
 }
 
+
 #define STEPSIZE sizeof(reg_t)
 LZ4_FORCE_INLINE
 unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
@@ -1288,22 +1306,6 @@ int LZ4_compress_default(const char* src, char* dst, int srcSize, int maxOutputS
 }
 
 
-/* hidden debug function */
-/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */
-int LZ4_compress_fast_force(const char* src, char* dst, int srcSize, int dstCapacity, int acceleration)
-{
-    LZ4_stream_t ctx;
-    LZ4_initStream(&ctx, sizeof(ctx));
-
-    if (srcSize < LZ4_64Klimit) {
-        return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, byU16,    noDict, noDictIssue, acceleration);
-    } else {
-        tableType_t const addrMode = (sizeof(void*) > 4) ? byU32 : byPtr;
-        return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, addrMode, noDict, noDictIssue, acceleration);
-    }
-}
-
-
 /* Note!: This function leaves the stream in an unclean/broken state!
  * It is not safe to subsequently use the same state with a _fastReset() or
  * _continue() call without resetting it. */
diff --git a/ossfuzz/travisoss.sh b/ossfuzz/travisoss.sh
index 5ea884c..eae9a80 100755
--- a/ossfuzz/travisoss.sh
+++ b/ossfuzz/travisoss.sh
@@ -12,7 +12,12 @@ then
 fi
 
 # Modify the oss-fuzz Dockerfile so that we're checking out the current branch on travis.
-sed -i "s@https://github.com/lz4/lz4.git@-b $TRAVIS_BRANCH https://github.com/lz4/lz4.git@" /tmp/ossfuzz/projects/lz4/Dockerfile
+if [ "x${TRAVIS_PULL_REQUEST}" = "xfalse" ]
+then
+    sed -i "s@https://github.com/lz4/lz4.git@-b ${TRAVIS_BRANCH} https://github.com/lz4/lz4.git@" /tmp/ossfuzz/projects/lz4/Dockerfile
+else
+    sed -i "s@https://github.com/lz4/lz4.git@-b ${TRAVIS_PULL_REQUEST_BRANCH} https://github.com/${TRAVIS_PULL_REQUEST_SLUG}.git@" /tmp/ossfuzz/projects/lz4/Dockerfile
+fi
 
 # Try and build the fuzzers
 pushd /tmp/ossfuzz
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 7926b20..0f3507e 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -680,7 +680,7 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress,
     prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)io_prefs->streamChecksum;
     prefs.favorDecSpeed = io_prefs->favorDecSpeed;
     if (io_prefs->contentSizeFlag) {
-      U64 const fileSize = UTIL_getFileSize(srcFileName);
+      U64 const fileSize = UTIL_getOpenFileSize(srcFile);
       prefs.frameInfo.contentSize = fileSize;   /* == 0 if input == stdin */
       if (fileSize==0)
           DISPLAYLEVEL(3, "Warning : cannot determine input content size \n");
@@ -1472,7 +1472,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
     LZ4IO_infoResult result = LZ4IO_format_not_known;  /* default result (error) */
     unsigned char buffer[LZ4F_HEADER_SIZE_MAX];
     FILE* const finput = LZ4IO_openSrcFile(input_filename);
-    cfinfo->fileSize = UTIL_getFileSize(input_filename);
+    cfinfo->fileSize = (finput == NULL) ? 0 : UTIL_getOpenFileSize(finput);
 
     while (!feof(finput)) {
         LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO;
diff --git a/programs/util.h b/programs/util.h
index 8e361ca..2f3d0a8 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -33,7 +33,7 @@ extern "C" {
 #include <stddef.h>       /* size_t, ptrdiff_t */
 #include <stdlib.h>       /* malloc */
 #include <string.h>       /* strlen, strncpy */
-#include <stdio.h>        /* fprintf */
+#include <stdio.h>        /* fprintf, fileno */
 #include <assert.h>
 #include <sys/types.h>    /* stat, utime */
 #include <sys/stat.h>     /* stat */
@@ -122,6 +122,36 @@ extern "C" {
 #endif
 
 
+/*-****************************************
+*  stat() functions
+******************************************/
+#if defined(_MSC_VER)
+#  define UTIL_TYPE_stat __stat64
+#  define UTIL_stat _stat64
+#  define UTIL_fstat _fstat64
+#  define UTIL_STAT_MODE_ISREG(st_mode) ((st_mode) & S_IFREG)
+#elif   defined(__MINGW32__) && defined (__MSVCRT__)
+#  define UTIL_TYPE_stat _stati64
+#  define UTIL_stat _stati64
+#  define UTIL_fstat _fstati64
+#  define UTIL_STAT_MODE_ISREG(st_mode) ((st_mode) & S_IFREG)
+#else
+#  define UTIL_TYPE_stat stat
+#  define UTIL_stat stat
+#  define UTIL_fstat fstat
+#  define UTIL_STAT_MODE_ISREG(st_mode) (S_ISREG(st_mode))
+#endif
+
+
+/*-****************************************
+*  fileno() function
+******************************************/
+#if defined(_MSC_VER)
+#  define UTIL_fileno _fileno
+#else
+#  define UTIL_fileno fileno
+#endif
+
 /* *************************************
 *  Constants
 ***************************************/
@@ -357,22 +387,30 @@ UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
 }
 
 
+UTIL_STATIC U64 UTIL_getOpenFileSize(FILE* file)
+{
+    int r;
+    int fd;
+    struct UTIL_TYPE_stat statbuf;
+
+    fd = UTIL_fileno(file);
+    if (fd < 0) {
+        perror("fileno");
+        exit(1);
+    }
+    r = UTIL_fstat(fd, &statbuf);
+    if (r || !UTIL_STAT_MODE_ISREG(statbuf.st_mode)) return 0;   /* No good... */
+    return (U64)statbuf.st_size;
+}
+
+
 UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
 {
     int r;
-#if defined(_MSC_VER)
-    struct __stat64 statbuf;
-    r = _stat64(infilename, &statbuf);
-    if (r || !(statbuf.st_mode & S_IFREG)) return 0;   /* No good... */
-#elif defined(__MINGW32__) && defined (__MSVCRT__)
-    struct _stati64 statbuf;
-    r = _stati64(infilename, &statbuf);
-    if (r || !(statbuf.st_mode & S_IFREG)) return 0;   /* No good... */
-#else
-    struct stat statbuf;
-    r = stat(infilename, &statbuf);
-    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
-#endif
+    struct UTIL_TYPE_stat statbuf;
+
+    r = UTIL_stat(infilename, &statbuf);
+    if (r || !UTIL_STAT_MODE_ISREG(statbuf.st_mode)) return 0;   /* No good... */
     return (U64)statbuf.st_size;
 }
 
diff --git a/tests/Makefile b/tests/Makefile
index 6404213..5a6ec59 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -106,7 +106,7 @@ checkFrame : lz4frame.o lz4.o lz4hc.o xxhash.o checkFrame.c
 
 decompress-partial: lz4.o decompress-partial.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
-	
+
 clean:
 	@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@@ -150,7 +150,9 @@ ifneq (,$(filter $(shell uname),SunOS))
 DIFF:=gdiff
 endif
 
+CAT:=cat
 DD:=dd
+DATAGEN:=./datagen
 
 .PHONY: list
 list:
@@ -172,14 +174,14 @@ lz4_all.o: lz4_all.c
 	$(CC) $(CFLAGS) $(CPPFLAGS) -c $^ -o $@
 
 lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c
-	cat $^ > $@
+	$(CAT) $^ > $@
 
 test-install: lz4 lib liblz4.pc
 	lz4_root=.. ./test_install.sh
 
 test-lz4-sparse: lz4 datagen
 	@echo "\n ---- test sparse file support ----"
-	./datagen -g5M  -P100 > tmplsdg5M
+	$(DATAGEN) -g5M  -P100 > tmplsdg5M
 	$(LZ4) -B4D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB4
 	$(DIFF) -s tmplsdg5M tmplscB4
 	$(LZ4) -B5D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB5
@@ -191,17 +193,17 @@ test-lz4-sparse: lz4 datagen
 	$(LZ4) tmplsdg5M -c | $(LZ4) -dv --no-sparse > tmplsnosparse
 	$(DIFF) -s tmplsdg5M tmplsnosparse
 	ls -ls tmpls*
-	./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd   # Odd size file (to generate non-full last block)
-	./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd
+	$(DATAGEN) -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd   # Odd size file (to generate non-full last block)
+	$(DATAGEN) -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd
 	ls -ls tmplsodd
 	@$(RM) tmpls*
 	@echo "\n Compatibility with Console :"
 	echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c
-	echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | cat
+	echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | $(CAT)
 	echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c
 	@echo "\n Compatibility with Append :"
-	./datagen -P100 -g1M > tmplsdg1M
-	cat tmplsdg1M tmplsdg1M > tmpls2M
+	$(DATAGEN) -P100 -g1M > tmplsdg1M
+	$(CAT) tmplsdg1M tmplsdg1M > tmpls2M
 	$(LZ4) -B5 -v tmplsdg1M tmplsc
 	$(LZ4) -d -v tmplsc tmplsr
 	$(LZ4) -d -v tmplsc -c >> tmplsr
@@ -211,20 +213,29 @@ test-lz4-sparse: lz4 datagen
 
 test-lz4-contentSize: lz4 datagen
 	@echo "\n ---- test original size support ----"
-	./datagen -g15M > tmplc1
+	$(DATAGEN) -g15M > tmplc1
 	$(LZ4) -v tmplc1 -c | $(LZ4) -t
 	$(LZ4) -v --content-size tmplc1 -c | $(LZ4) -d > tmplc2
-	$(DIFF) -s tmplc1 tmplc2
+	$(DIFF) tmplc1 tmplc2
+	$(LZ4) -f tmplc1 -c > tmplc1.lz4
+	$(LZ4) --content-size tmplc1 -c > tmplc2.lz4
+	! $(DIFF) tmplc1.lz4 tmplc2.lz4  # must differ, due to content size
+	$(LZ4) --content-size < tmplc1 > tmplc3.lz4
+	$(DIFF) tmplc2.lz4 tmplc3.lz4  # both must contain content size
+	$(CAT) tmplc1 | $(LZ4) > tmplc4.lz4
+	$(DIFF) tmplc1.lz4 tmplc4.lz4  # both don't have content size
+	$(CAT) tmplc1 | $(LZ4) --content-size > tmplc5.lz4 # can't determine content size
+	$(DIFF) tmplc1.lz4 tmplc5.lz4  # both don't have content size
 	@$(RM) tmplc*
 
 test-lz4-frame-concatenation: lz4 datagen
 	@echo "\n ---- test frame concatenation ----"
 	@echo -n > tmp-lfc-empty
 	@echo hi > tmp-lfc-nonempty
-	cat tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src
+	$(CAT) tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src
 	$(LZ4) -zq tmp-lfc-empty -c > tmp-lfc-empty.lz4
 	$(LZ4) -zq tmp-lfc-nonempty -c > tmp-lfc-nonempty.lz4
-	cat tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4
+	$(CAT) tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4
 	$(LZ4) -d tmp-lfc-concat.lz4 -c > tmp-lfc-result
 	$(CMP) tmp-lfc-src tmp-lfc-result
 	@$(RM) tmp-lfc-*
@@ -232,9 +243,9 @@ test-lz4-frame-concatenation: lz4 datagen
 
 test-lz4-multiple: lz4 datagen
 	@echo "\n ---- test multiple files ----"
-	@./datagen -s1        > tmp-tlm1 2> $(VOID)
-	@./datagen -s2 -g100K > tmp-tlm2 2> $(VOID)
-	@./datagen -s3 -g200K > tmp-tlm3 2> $(VOID)
+	@$(DATAGEN) -s1        > tmp-tlm1 2> $(VOID)
+	@$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID)
+	@$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID)
 	# compress multiple files : one .lz4 per source file
 	$(LZ4) -f -m tmp-tlm*
 	test -f tmp-tlm1.lz4
@@ -249,7 +260,7 @@ test-lz4-multiple: lz4 datagen
 	$(CMP) tmp-tlm2 tmp-tlm2-orig
 	$(CMP) tmp-tlm3 tmp-tlm3-orig
 	# compress multiple files into stdout
-	cat tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
+	$(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
 	$(RM) *.lz4
 	$(LZ4) -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2
 	test ! -f tmp-tlm1.lz4  # must not create .lz4 artefact
@@ -257,7 +268,7 @@ test-lz4-multiple: lz4 datagen
 	# decompress multiple files into stdout
 	$(RM) tmp-tlm-concat1 tmp-tlm-concat2
 	$(LZ4) -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3   # generate .lz4 to decompress
-	cat tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1   # create concatenated reference
+	$(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1   # create concatenated reference
 	$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
 	$(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2
 	test ! -f tmp-tlm1  # must not create file artefact
@@ -268,9 +279,9 @@ test-lz4-multiple: lz4 datagen
 
 test-lz4-multiple-legacy: lz4 datagen
 	@echo "\n ---- test multiple files (Legacy format) ----"
-	@./datagen -s1        > tmp-tlm1 2> $(VOID)
-	@./datagen -s2 -g100K > tmp-tlm2 2> $(VOID)
-	@./datagen -s3 -g200K > tmp-tlm3 2> $(VOID)
+	@$(DATAGEN) -s1        > tmp-tlm1 2> $(VOID)
+	@$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID)
+	@$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID)
 	# compress multiple files using legacy format: one .lz4 per source file
 	$(LZ4) -f -l -m tmp-tlm*
 	test -f tmp-tlm1.lz4
@@ -286,7 +297,7 @@ test-lz4-multiple-legacy: lz4 datagen
 	$(CMP) tmp-tlm2 tmp-tlm2-orig
 	$(CMP) tmp-tlm3 tmp-tlm3-orig
 	# compress multiple files into stdout using legacy format
-	cat tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
+	$(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
 	$(RM) *.lz4
 	$(LZ4) -l -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2
 	test ! -f tmp-tlm1.lz4  # must not create .lz4 artefact
@@ -294,7 +305,7 @@ test-lz4-multiple-legacy: lz4 datagen
 	# # # decompress multiple files into stdout using legacy format
 	$(RM) tmp-tlm-concat1 tmp-tlm-concat2
 	$(LZ4) -l -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3   # generate .lz4 to decompress
-	cat tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1   # create concatenated reference
+	$(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1   # create concatenated reference
 	$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
 	$(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2
 	$(LZ4) -d -l -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 # -l mustn't impact option -d
@@ -306,18 +317,18 @@ test-lz4-multiple-legacy: lz4 datagen
 
 test-lz4-basic: lz4 datagen unlz4 lz4cat
 	@echo "\n ---- test lz4 basic compression/decompression ----"
-	./datagen -g0       | $(LZ4) -v     | $(LZ4) -t
-	./datagen -g16KB    | $(LZ4) -9     | $(LZ4) -t
-	./datagen -g20KB > tmp-tlb-dg20k
+	$(DATAGEN) -g0       | $(LZ4) -v     | $(LZ4) -t
+	$(DATAGEN) -g16KB    | $(LZ4) -9     | $(LZ4) -t
+	$(DATAGEN) -g20KB > tmp-tlb-dg20k
 	$(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
 	$(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
 	$(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
 	$(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
-	./datagen           | $(LZ4) -BI    | $(LZ4) -t
-	./datagen -g6M -P99 | $(LZ4) -9BD   | $(LZ4) -t
-	./datagen -g17M     | $(LZ4) -9v    | $(LZ4) -qt
-	./datagen -g33M     | $(LZ4) --no-frame-crc | $(LZ4) -t
-	./datagen -g256MB   | $(LZ4) -vqB4D | $(LZ4) -t
+	$(DATAGEN)           | $(LZ4) -BI    | $(LZ4) -t
+	$(DATAGEN) -g6M -P99 | $(LZ4) -9BD   | $(LZ4) -t
+	$(DATAGEN) -g17M     | $(LZ4) -9v    | $(LZ4) -qt
+	$(DATAGEN) -g33M     | $(LZ4) --no-frame-crc | $(LZ4) -t
+	$(DATAGEN) -g256MB   | $(LZ4) -vqB4D | $(LZ4) -t
 	@echo "hello world" > tmp-tlb-hw
 	$(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4
 	test ! -f tmp-tlb-hw                      # must fail (--rm)
@@ -346,18 +357,18 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
 	$(DIFF) -q tmp-tlb-hw tmp-tlb4
 	$(LZ4) -f tmp-tlb-hw
 	$(LZ4) --list tmp-tlb-hw.lz4           # test --list on valid single-frame file
-	cat tmp-tlb-hw >> tmp-tlb-hw.lz4
+	$(CAT) tmp-tlb-hw >> tmp-tlb-hw.lz4
 	$(LZ4) -f tmp-tlb-hw.lz4               # uncompress valid frame followed by invalid data
 	$(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv  # test block checksum
-	# ./datagen -g20KB generates the same file every single time
-	# cannot save output of ./datagen -g20KB as input file to lz4 because the following shell commands are run before ./datagen -g20KB
-	test "$(shell ./datagen -g20KB | $(LZ4) -c --fast | wc -c)" -lt "$(shell ./datagen -g20KB | $(LZ4) -c --fast=9 | wc -c)" # -1 vs -9
-	test "$(shell ./datagen -g20KB | $(LZ4) -c -1 | wc -c)" -lt "$(shell ./datagen -g20KB| $(LZ4) -c --fast=1 | wc -c)" # 1 vs -1
-	test "$(shell ./datagen -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell ./datagen -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1
+	# $(DATAGEN) -g20KB generates the same file every single time
+	# cannot save output of $(DATAGEN) -g20KB as input file to lz4 because the following shell commands are run before $(DATAGEN) -g20KB
+	test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast | wc -c)" -lt "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=9 | wc -c)" # -1 vs -9
+	test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c -1 | wc -c)" -lt "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast=1 | wc -c)" # 1 vs -1
+	test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1
 	! $(LZ4) -c --fast=0 tmp-tlb-dg20K # lz4 should fail when fast=0
 	! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1
 	# High --fast values can result in out-of-bound dereferences #876
-	./datagen -g1M | $(LZ4) -c --fast=999999999 > /dev/null
+	$(DATAGEN) -g1M | $(LZ4) -c --fast=999999999 > /dev/null
 	# Test for #596
 	@echo "TEST" > tmp-tlb-test
 	$(LZ4) -m tmp-tlb-test
@@ -369,10 +380,10 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
 
 test-lz4-dict: lz4 datagen
 	@echo "\n ---- test lz4 compression/decompression with dictionary ----"
-	./datagen -g16KB > tmp-dict
-	./datagen -g32KB > tmp-dict-sample-32k
+	$(DATAGEN) -g16KB > tmp-dict
+	$(DATAGEN) -g32KB > tmp-dict-sample-32k
 	< tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k
-	./datagen -g128MB > tmp-dict-sample-128m
+	$(DATAGEN) -g128MB > tmp-dict-sample-128m
 	< tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m
 	touch tmp-dict-sample-0
 	< tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0
@@ -381,10 +392,10 @@ test-lz4-dict: lz4 datagen
 	< tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0
 
 	@echo "\n ---- test lz4 dictionary loading ----"
-	./datagen -g128KB > tmp-dict-data-128KB
+	$(DATAGEN) -g128KB > tmp-dict-data-128KB
 	set -e; \
 	for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \
-		./datagen -g$$l > tmp-dict-$$l; \
+		$(DATAGEN) -g$$l > tmp-dict-$$l; \
 		$(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \
 		< tmp-dict-$$l      $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \
 		< tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l      | $(DIFF) - tmp-dict-data-128KB; \
@@ -394,12 +405,12 @@ test-lz4-dict: lz4 datagen
 
 test-lz4-hugefile: lz4 datagen
 	@echo "\n ---- test huge files compression/decompression ----"
-	./datagen -g6GB   | $(LZ4) -vB5D  | $(LZ4) -qt
+	./datagen -g6GB    | $(LZ4) -vB5D  | $(LZ4) -qt
 	./datagen -g4500MB | $(LZ4) -v3BD | $(LZ4) -qt
 	# test large file size [2-4] GB
-	@./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1
+	@$(DATAGEN) -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1
 	@ls -ls tmphf1
-	@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmphf2
+	@$(DATAGEN) -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmphf2
 	@ls -ls tmphf2
 	$(DIFF) -s tmphf1 tmphf2
 	@$(RM) tmphf*
@@ -408,14 +419,14 @@ test-lz4-testmode: lz4 datagen
 	@echo "\n ---- bench mode ----"
 	$(LZ4) -bi1
 	@echo "\n ---- test mode ----"
-	! ./datagen | $(LZ4) -t
-	! ./datagen | $(LZ4) -tf
+	! $(DATAGEN) | $(LZ4) -t
+	! $(DATAGEN) | $(LZ4) -tf
 	@echo "\n ---- pass-through mode ----"
 	@echo "Why hello there " > tmp-tlt2.lz4
 	! $(LZ4) -f tmp-tlt2.lz4 > $(VOID)
-	! ./datagen | $(LZ4) -dc  > $(VOID)
-	! ./datagen | $(LZ4) -df > $(VOID)
-	./datagen | $(LZ4) -dcf > $(VOID)
+	! $(DATAGEN) | $(LZ4) -dc  > $(VOID)
+	! $(DATAGEN) | $(LZ4) -df > $(VOID)
+	$(DATAGEN) | $(LZ4) -dcf > $(VOID)
 	@echo "Hello World !" > tmp-tlt1
 	$(LZ4) -dcf tmp-tlt1
 	@echo "from underground..." > tmp-tlt2
@@ -429,16 +440,16 @@ test-lz4-testmode: lz4 datagen
 
 test-lz4-opt-parser: lz4 datagen
 	@echo "\n ---- test opt-parser ----"
-	./datagen -g16KB      | $(LZ4) -12      | $(LZ4) -t
-	./datagen -P10        | $(LZ4) -12B4    | $(LZ4) -t
-	./datagen -g256K      | $(LZ4) -12B4D   | $(LZ4) -t
-	./datagen -g512K -P25 | $(LZ4) -12BD    | $(LZ4) -t
-	./datagen -g1M        | $(LZ4) -12B5    | $(LZ4) -t
-	./datagen -g2M -P99   | $(LZ4) -11B4D   | $(LZ4) -t
-	./datagen -g4M        | $(LZ4) -11vq    | $(LZ4) -qt
-	./datagen -g8M        | $(LZ4) -11B4    | $(LZ4) -t
-	./datagen -g16M -P90  | $(LZ4) -11B5    | $(LZ4) -t
-	./datagen -g32M -P10  | $(LZ4) -11B5D   | $(LZ4) -t
+	$(DATAGEN) -g16KB      | $(LZ4) -12      | $(LZ4) -t
+	$(DATAGEN) -P10        | $(LZ4) -12B4    | $(LZ4) -t
+	$(DATAGEN) -g256K      | $(LZ4) -12B4D   | $(LZ4) -t
+	$(DATAGEN) -g512K -P25 | $(LZ4) -12BD    | $(LZ4) -t
+	$(DATAGEN) -g1M        | $(LZ4) -12B5    | $(LZ4) -t
+	$(DATAGEN) -g2M -P99   | $(LZ4) -11B4D   | $(LZ4) -t
+	$(DATAGEN) -g4M        | $(LZ4) -11vq    | $(LZ4) -qt
+	$(DATAGEN) -g8M        | $(LZ4) -11B4    | $(LZ4) -t
+	$(DATAGEN) -g16M -P90  | $(LZ4) -11B5    | $(LZ4) -t
+	$(DATAGEN) -g32M -P10  | $(LZ4) -11B5D   | $(LZ4) -t
 
 test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-multiple-legacy \
                       test-lz4-frame-concatenation test-lz4-testmode \
@@ -451,35 +462,35 @@ test-lz4: lz4 datagen test-lz4-essentials test-lz4-opt-parser \
 
 test-lz4c: lz4c datagen
 	@echo "\n ---- test lz4c variant ----"
-	./datagen -g256MB | $(LZ4)c -l -v    | $(LZ4)c   -t
+	$(DATAGEN) -g256MB | $(LZ4)c -l -v    | $(LZ4)c   -t
 
 test-lz4c32: CFLAGS+=-m32
 test-lz4c32: test-lz4
 
 test-interop-32-64: lz4 lz4c32 datagen
 	@echo "\n ---- test interoperability 32-bits -vs- 64 bits ----"
-	./datagen -g16KB  | $(LZ4)c32 -9     | $(LZ4)    -t
-	./datagen -P10    | $(LZ4)    -9B4   | $(LZ4)c32 -t
-	./datagen         | $(LZ4)c32        | $(LZ4)    -t
-	./datagen -g1M    | $(LZ4)    -3B5   | $(LZ4)c32 -t
-	./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)    -qt
-	./datagen -g1G -P90 | $(LZ4)         | $(LZ4)c32 -t
-	./datagen -g6GB   | $(LZ4)c32 -vq9BD | $(LZ4)    -qt
+	$(DATAGEN) -g16KB  | $(LZ4)c32 -9     | $(LZ4)    -t
+	$(DATAGEN) -P10    | $(LZ4)    -9B4   | $(LZ4)c32 -t
+	$(DATAGEN)         | $(LZ4)c32        | $(LZ4)    -t
+	$(DATAGEN) -g1M    | $(LZ4)    -3B5   | $(LZ4)c32 -t
+	$(DATAGEN) -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)    -qt
+	$(DATAGEN) -g1G -P90 | $(LZ4)         | $(LZ4)c32 -t
+	$(DATAGEN) -g6GB   | $(LZ4)c32 -vq9BD | $(LZ4)    -qt
 
 test-lz4c32-basic: lz4c32 datagen
 	@echo "\n ---- test lz4c32 32-bits version ----"
-	./datagen -g16KB  | $(LZ4)c32 -9     | $(LZ4)c32 -t
-	./datagen         | $(LZ4)c32        | $(LZ4)c32 -t
-	./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)c32 -qt
-	./datagen -g6GB   | $(LZ4)c32 -vqB5D | $(LZ4)c32 -qt
+	$(DATAGEN) -g16KB  | $(LZ4)c32 -9     | $(LZ4)c32 -t
+	$(DATAGEN)         | $(LZ4)c32        | $(LZ4)c32 -t
+	$(DATAGEN) -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)c32 -qt
+	$(DATAGEN) -g6GB   | $(LZ4)c32 -vqB5D | $(LZ4)c32 -qt
 
 test-platform:
 	@echo "\n ---- test lz4 $(QEMU_SYS) platform ----"
-	$(QEMU_SYS) ./datagen -g16KB  | $(QEMU_SYS) $(LZ4) -9     | $(QEMU_SYS) $(LZ4) -t
-	$(QEMU_SYS) ./datagen         | $(QEMU_SYS) $(LZ4)        | $(QEMU_SYS) $(LZ4) -t
-	$(QEMU_SYS) ./datagen -g256MB | $(QEMU_SYS) $(LZ4) -vqB4D | $(QEMU_SYS) $(LZ4) -qt
+	$(QEMU_SYS) $(DATAGEN) -g16KB  | $(QEMU_SYS) $(LZ4) -9     | $(QEMU_SYS) $(LZ4) -t
+	$(QEMU_SYS) $(DATAGEN)         | $(QEMU_SYS) $(LZ4)        | $(QEMU_SYS) $(LZ4) -t
+	$(QEMU_SYS) $(DATAGEN) -g256MB | $(QEMU_SYS) $(LZ4) -vqB4D | $(QEMU_SYS) $(LZ4) -qt
 ifneq ($(QEMU_SYS),qemu-arm-static)
-	$(QEMU_SYS) ./datagen -g3GB   | $(QEMU_SYS) $(LZ4) -vqB5D | $(QEMU_SYS) $(LZ4) -qt
+	$(QEMU_SYS) $(DATAGEN) -g3GB   | $(QEMU_SYS) $(LZ4) -vqB5D | $(QEMU_SYS) $(LZ4) -qt
 endif
 
 test-fullbench: fullbench
@@ -502,13 +513,13 @@ test-frametest32: test-frametest
 
 test-mem: lz4 datagen fuzzer frametest fullbench
 	@echo "\n ---- valgrind tests : memory analyzer ----"
-	valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
-	./datagen -g16KB > ftmdg16K
+	valgrind --leak-check=yes --error-exitcode=1 $(DATAGEN) -g50M > $(VOID)
+	$(DATAGEN) -g16KB > ftmdg16K
 	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID)
-	./datagen -g16KB -s2 > ftmdg16K2
-	./datagen -g16KB -s3 > ftmdg16K3
+	$(DATAGEN) -g16KB -s2 > ftmdg16K2
+	$(DATAGEN) -g16KB -s3 > ftmdg16K3
 	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3
-	./datagen -g7MB > ftmdg7M
+	$(DATAGEN) -g7MB > ftmdg7M
 	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2
 	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2
 	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M