Merge pull request #252 from inikep/dev

Dev

9 files changed, 966 insertions, 342 deletions

diff --git a/.travis.yml b/.travis.yml
index 2a0eaab..ef8732c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -100,6 +100,8 @@ matrix:
           sources:
             - ubuntu-toolchain-r-test
           packages:
+            - libc6-dev-i386
+            - gcc-multilib
             - gcc-5
             - gcc-5-multilib
     - env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=gcc-6' COMPILER=gcc-6
diff --git a/Makefile b/Makefile
index 6997b19..e98aafd 100644
--- a/Makefile
+++ b/Makefile
@@ -30,8 +30,9 @@
 #  - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
 # ################################################################
 
-DESTDIR?=
-PREFIX ?= /usr/local
+DESTDIR ?=
+PREFIX  ?= /usr/local
+VOID    := /dev/null
 
 LIBDIR ?= $(PREFIX)/lib
 INCLUDEDIR=$(PREFIX)/include
@@ -43,10 +44,8 @@ TESTDIR = tests
 # Define nul output
 ifneq (,$(filter Windows%,$(OS)))
 EXT = .exe
-VOID = nul
 else
 EXT =
-VOID = /dev/null
 endif
 
 
diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt
index 6edec98..899ce35 100644
--- a/contrib/cmake_unofficial/CMakeLists.txt
+++ b/contrib/cmake_unofficial/CMakeLists.txt
@@ -27,7 +27,7 @@ ENDIF()
 set(LZ4_DIR ../../lib/)
 set(PRG_DIR ../../programs/)
 set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h ${LZ4_DIR}lz4frame.c ${LZ4_DIR}lz4frame.h ${LZ4_DIR}xxhash.c)
-set(LZ4_SRCS ${LZ4_DIR}lz4frame.c ${LZ4_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c)
+set(LZ4_SRCS ${LZ4_DIR}lz4frame.c ${LZ4_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c ${PRG_DIR}datagen.c)
 
 if(BUILD_TOOLS AND NOT (LINK_TOOLS_WITH_LIB AND BUILD_LIBS))
     set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB})
diff --git a/programs/Makefile b/programs/Makefile
index ba01b40..39672b3 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -29,6 +29,7 @@
 
 DESTDIR ?=
 PREFIX  ?= /usr/local
+VOID    := /dev/null
 BINDIR  := $(PREFIX)/bin
 MANDIR  := $(PREFIX)/share/man/man1
 LZ4DIR  := ../lib
@@ -43,10 +44,8 @@ FLAGS   := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
 EXT =.exe
-VOID = nul
 else
 EXT =
-VOID = /dev/null
 endif
 
 
@@ -59,13 +58,13 @@ bins: lz4 lz4c
 
 all: bins m32
 
-lz4: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o
+lz4: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-lz4c  : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o
+lz4c  : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o
 	$(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
 
-lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c
+lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c datagen.c
 	$(CC) -m32 $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
 
 clean:
diff --git a/programs/bench.c b/programs/bench.c
index bec93f9..33f5fcf 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -23,36 +23,18 @@
     - LZ4 source repository : https://github.com/lz4/lz4
 */
 
-/*-************************************
-*  Compiler Options
-***************************************/
-#if defined(_MSC_VER) || defined(_WIN32)
-#  define _CRT_SECURE_NO_WARNINGS
-#  define _CRT_SECURE_NO_DEPRECATE     /* VS2005 */
-#endif
-
-/* Unix Large Files support (>4GB) */
-#define _FILE_OFFSET_BITS 64
-#if (defined(__sun__) && (!defined(__LP64__)))   /* Sun Solaris 32-bits requires specific definitions */
-#  define _LARGEFILE_SOURCE
-#elif ! defined(__LP64__)                        /* No point defining Large file for 64 bit */
-#  define _LARGEFILE64_SOURCE
-#endif
-
-#if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
-#  define _POSIX_SOURCE 1          /* disable %llu warnings with MinGW on Windows */
-#endif
 
-
-/*-************************************
+/* *************************************
 *  Includes
 ***************************************/
-#include <stdlib.h>     /* malloc */
-#include <stdio.h>      /* fprintf, fopen */
-#include <sys/types.h>  /* stat64 */
-#include <sys/stat.h>   /* stat64 */
-#include <time.h>       /* clock_t, clock, CLOCKS_PER_SEC */
-#include <string.h>     /* strlen */
+#include "util.h"        /* Compiler options, UTIL_GetFileSize, UTIL_sleep */
+#include <stdlib.h>      /* malloc, free */
+#include <string.h>      /* memset */
+#include <stdio.h>       /* fprintf, fopen, ftello64 */
+#include <time.h>        /* clock_t, clock, CLOCKS_PER_SEC */
+
+#include "datagen.h"     /* RDG_genBuffer */
+#include "xxhash.h"
 
 
 #include "lz4.h"
@@ -61,119 +43,327 @@ static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSi
 #include "lz4hc.h"
 #define COMPRESSOR1 LZ4_compress_HC
 #define DEFAULTCOMPRESSOR COMPRESSOR0
-
-#include "xxhash.h"
+#define LZ4_isError(errcode) (errcode==0) 
 
 
-/*-************************************
-*  Compiler specifics
-***************************************/
-#if !defined(S_ISREG)
-#  define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
-#endif
-
-
-/*-************************************
-*  Basic Types
+/* *************************************
+*  Constants
 ***************************************/
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
-# include <stdint.h>
-  typedef uint8_t  BYTE;
-  typedef uint16_t U16;
-  typedef uint32_t U32;
-  typedef  int32_t S32;
-  typedef uint64_t U64;
+#ifndef LZ4_GIT_COMMIT_STRING
+#  define LZ4_GIT_COMMIT_STRING ""
 #else
-  typedef unsigned char       BYTE;
-  typedef unsigned short      U16;
-  typedef unsigned int        U32;
-  typedef   signed int        S32;
-  typedef unsigned long long  U64;
+#  define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT)
 #endif
 
-
-/*-************************************
-*  Constants
-***************************************/
-#define NBLOOPS    3
-#define TIMELOOP_S 1
-#define TIMELOOP_CLOCK (TIMELOOP_S * CLOCKS_PER_SEC)
+#define NBSECONDS             3
+#define TIMELOOP_MICROSEC     1*1000000ULL /* 1 second */
+#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
+#define COOLPERIOD_SEC        10
+#define DECOMP_MULT           2 /* test decompression DECOMP_MULT times longer than compression */
 
 #define KB *(1 <<10)
 #define MB *(1 <<20)
 #define GB *(1U<<30)
 
-#define MAX_MEM             (2 GB - 64 MB)
-#define DEFAULT_CHUNKSIZE   (4 MB)
+static const size_t maxMemory = (sizeof(size_t)==4)  ?  (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
+
+static U32 g_compressibilityDefault = 50;
 
 
-/*-************************************
-*  Local structures
+/* *************************************
+*  console display
 ***************************************/
-struct chunkParameters
-{
-    U32   id;
-    char* origBuffer;
-    char* compressedBuffer;
-    int   origSize;
-    int   compressedSize;
-};
+#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = 2;   /* 0 : no display;   1: errors;   2 : + result + interaction + warnings;   3 : + progression;   4 : + information */
 
-struct compressionParameters
-{
-    int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
-    int (*decompressionFunction)(const char* src, char* dst, int dstSize);
-};
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+            if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
+            { g_time = clock(); DISPLAY(__VA_ARGS__); \
+            if (g_displayLevel>=4) fflush(stdout); } }
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
 
 
-/*-************************************
-*  Macro
+/* *************************************
+*  Exceptions
 ***************************************/
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#ifndef DEBUG
+#  define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...)                                             \
+{                                                                         \
+    DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+    DISPLAYLEVEL(1, "Error %i : ", error);                                \
+    DISPLAYLEVEL(1, __VA_ARGS__);                                         \
+    DISPLAYLEVEL(1, "\n");                                                \
+    exit(error);                                                          \
+}
 
 
-/*-************************************
+/* *************************************
 *  Benchmark Parameters
 ***************************************/
-static int g_chunkSize = DEFAULT_CHUNKSIZE;
-static int g_nbIterations = NBLOOPS;
-static int BMK_pause = 0;
+static U32 g_nbSeconds = NBSECONDS;
+static size_t g_blockSize = 0;
+int g_additionalParam = 0;
 
-void BMK_setBlocksize(int bsize) { g_chunkSize = bsize; }
+void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 
-void BMK_setNbIterations(int nbLoops)
+void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
+
+void BMK_SetNbSeconds(unsigned nbSeconds)
 {
-    g_nbIterations = nbLoops;
-    DISPLAY("- %i iterations -\n", g_nbIterations);
+    g_nbSeconds = nbSeconds;
+    DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
 }
 
-void BMK_setPause(void) { BMK_pause = 1; }
+void BMK_SetBlockSize(size_t blockSize)
+{
+    g_blockSize = blockSize;
+    DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
+}
 
 
-/*-*******************************************************
-*  Private functions
+/* ********************************************************
+*  Bench functions
 **********************************************************/
+typedef struct
+{
+    const char* srcPtr;
+    size_t srcSize;
+    char*  cPtr;
+    size_t cRoom;
+    size_t cSize;
+    char*  resPtr;
+    size_t resSize;
+} blockParam_t;
 
-/** BMK_getClockSpan() :
-    works even if overflow; Typical max span ~ 30 mn */
-static clock_t BMK_getClockSpan (clock_t clockStart)
+struct compressionParameters
 {
-    return clock() - clockStart;
+    int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
+};
+
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+#define MAX(a,b) ((a)>(b) ? (a) : (b))
+
+static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
+                        const char* displayName, int cLevel,
+                        const size_t* fileSizes, U32 nbFiles)
+{
+    size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
+    U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
+    blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
+    size_t const maxCompressedSize = LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024);   /* add some room for safety */
+    void* const compressedBuffer = malloc(maxCompressedSize);
+    void* const resultBuffer = malloc(srcSize);
+    U32 nbBlocks;
+    UTIL_time_t ticksPerSecond;
+    struct compressionParameters compP;
+    int cfunctionId;
+
+    /* checks */
+    if (!compressedBuffer || !resultBuffer || !blockTable)
+        EXM_THROW(31, "allocation error : not enough memory");
+
+    /* init */
+    if (strlen(displayName)>17) displayName += strlen(displayName)-17;   /* can only display 17 characters */
+    UTIL_initTimer(&ticksPerSecond);
+
+    /* Init */
+    if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1;
+    switch (cfunctionId)
+    {
+#ifdef COMPRESSOR0
+    case 0 : compP.compressionFunction = COMPRESSOR0; break;
+#endif
+#ifdef COMPRESSOR1
+    case 1 : compP.compressionFunction = COMPRESSOR1; break;
+#endif
+    default : compP.compressionFunction = DEFAULTCOMPRESSOR;
+    }
+
+    /* Init blockTable data */
+    {   const char* srcPtr = (const char*)srcBuffer;
+        char* cPtr = (char*)compressedBuffer;
+        char* resPtr = (char*)resultBuffer;
+        U32 fileNb;
+        for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
+            size_t remaining = fileSizes[fileNb];
+            U32 const nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
+            U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
+            for ( ; nbBlocks<blockEnd; nbBlocks++) {
+                size_t const thisBlockSize = MIN(remaining, blockSize);
+                blockTable[nbBlocks].srcPtr = srcPtr;
+                blockTable[nbBlocks].cPtr = cPtr;
+                blockTable[nbBlocks].resPtr = resPtr;
+                blockTable[nbBlocks].srcSize = thisBlockSize;
+                blockTable[nbBlocks].cRoom = LZ4_compressBound((int)thisBlockSize);
+                srcPtr += thisBlockSize;
+                cPtr += blockTable[nbBlocks].cRoom;
+                resPtr += thisBlockSize;
+                remaining -= thisBlockSize;
+    }   }   }
+
+    /* warmimg up memory */
+    RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
+
+    /* Bench */
+    {   U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
+        U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
+        UTIL_time_t coolTime;
+        U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 100;
+        U64 totalCTime=0, totalDTime=0;
+        U32 cCompleted=0, dCompleted=0;
+#       define NB_MARKS 4
+        const char* const marks[NB_MARKS] = { " |", " /", " =",  "\\" };
+        U32 markNb = 0;
+        size_t cSize = 0;
+        double ratio = 0.;
+
+        UTIL_getTime(&coolTime);
+        DISPLAYLEVEL(2, "\r%79s\r", "");
+        while (!cCompleted | !dCompleted) {
+            UTIL_time_t clockStart;
+            U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
+
+            /* overheat protection */
+            if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) {
+                DISPLAYLEVEL(2, "\rcooling down ...    \r");
+                UTIL_sleep(COOLPERIOD_SEC);
+                UTIL_getTime(&coolTime);
+            }
+
+            /* Compression */
+            DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
+            if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize);  /* warm up and erase result buffer */
+
+            UTIL_sleepMilli(1);  /* give processor time to other processes */
+            UTIL_waitForNextTick(ticksPerSecond);
+            UTIL_getTime(&clockStart);
+
+            if (!cCompleted) {   /* still some time to do compression tests */
+                U32 nbLoops = 0;
+                do {
+                    U32 blockNb;
+                    for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+                        size_t const rSize = compP.compressionFunction(blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr, (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom, cLevel);
+                        if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4_compress() failed");
+                        blockTable[blockNb].cSize = rSize;
+                    }
+                    nbLoops++;
+                } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+                {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
+                    if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
+                    totalCTime += clockSpan;
+                    cCompleted = totalCTime>maxTime;
+            }   }
+
+            cSize = 0;
+            { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
+            ratio = (double)srcSize / (double)cSize;
+            markNb = (markNb+1) % NB_MARKS;
+            DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
+                    marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
+                    (double)srcSize / fastestC );
+
+            (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
+#if 1
+            /* Decompression */
+            if (!dCompleted) memset(resultBuffer, 0xD6, srcSize);  /* warm result buffer */
+
+            UTIL_sleepMilli(1); /* give processor time to other processes */
+            UTIL_waitForNextTick(ticksPerSecond);
+            UTIL_getTime(&clockStart);
+
+            if (!dCompleted) {
+                U32 nbLoops = 0;
+                do {
+                    U32 blockNb;
+                    for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+                        size_t const regenSize = LZ4_decompress_safe(blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize);
+                        if (LZ4_isError(regenSize)) {
+                            DISPLAY("LZ4_decompress_safe() failed on block %u  \n", blockNb);
+                            clockLoop = 0;   /* force immediate test end */
+                            break;
+                        }
+
+                        blockTable[blockNb].resSize = regenSize;
+                    }
+                    nbLoops++;
+                } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < DECOMP_MULT*clockLoop);
+                {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
+                    if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
+                    totalDTime += clockSpan;
+                    dCompleted = totalDTime>(DECOMP_MULT*maxTime);
+            }   }
+
+            markNb = (markNb+1) % NB_MARKS;
+            DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
+                    marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
+                    (double)srcSize / fastestC,
+                    (double)srcSize / fastestD );
+
+            /* CRC Checking */
+            {   U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
+                if (crcOrig!=crcCheck) {
+                    size_t u;
+                    DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x   \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
+                    for (u=0; u<srcSize; u++) {
+                        if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
+                            U32 segNb, bNb, pos;
+                            size_t bacc = 0;
+                            DISPLAY("Decoding error at pos %u ", (U32)u);
+                            for (segNb = 0; segNb < nbBlocks; segNb++) {
+                                if (bacc + blockTable[segNb].srcSize > u) break;
+                                bacc += blockTable[segNb].srcSize;
+                            }
+                            pos = (U32)(u - bacc);
+                            bNb = pos / (128 KB);
+                            DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
+                            break;
+                        }
+                        if (u==srcSize-1) {  /* should never happen */
+                            DISPLAY("no difference detected\n");
+                    }   }
+                    break;
+            }   }   /* CRC Checking */
+#endif
+        }   /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
+
+        if (g_displayLevel == 1) {
+            double cSpeed = (double)srcSize / fastestC;
+            double dSpeed = (double)srcSize / fastestD;
+            if (g_additionalParam)
+                DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
+            else
+                DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
+        }
+        DISPLAYLEVEL(2, "%2i#\n", cLevel);
+    }   /* Bench */
+
+    /* clean up */
+    free(blockTable);
+    free(compressedBuffer);
+    free(resultBuffer);
+    return 0;
 }
 
+
 static size_t BMK_findMaxMem(U64 requiredMem)
 {
-    size_t const step = 64 MB;
-    void* testmem = NULL;
+    size_t step = 64 MB;
+    BYTE* testmem=NULL;
 
     requiredMem = (((requiredMem >> 26) + 1) << 26);
     requiredMem += 2*step;
-    if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
+    if (requiredMem > maxMemory) requiredMem = maxMemory;
 
     while (!testmem) {
         if (requiredMem > step) requiredMem -= step;
         else requiredMem >>= 1;
-        testmem = malloc ((size_t)requiredMem);
+        testmem = (BYTE*) malloc ((size_t)requiredMem);
     }
     free (testmem);
 
@@ -184,225 +374,130 @@ static size_t BMK_findMaxMem(U64 requiredMem)
     return (size_t)requiredMem;
 }
 
-static U64 BMK_GetFileSize(const char* infilename)
+
+static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
+                            const char* displayName, int cLevel, int cLevelLast,
+                            const size_t* fileSizes, unsigned nbFiles)
 {
-    int r;
-#if defined(_MSC_VER)
-    struct _stat64 statbuf;
-    r = _stat64(infilename, &statbuf);
-#else
-    struct stat statbuf;
-    r = stat(infilename, &statbuf);
-#endif
-    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
-    return (U64)statbuf.st_size;
-}
+    int l;
 
+    const char* pch = strrchr(displayName, '\\'); /* Windows */
+    if (!pch) pch = strrchr(displayName, '/'); /* Linux */
+    if (pch) displayName = pch+1;
 
-/*-*******************************************************
-*  Public function
-**********************************************************/
+    SET_HIGH_PRIORITY;
 
-int BMK_benchLevel(const char** fileNamesTable, int nbFiles, int cLevel)
-{
-  int fileIdx=0;
-  char* orig_buff;
-  struct compressionParameters compP;
-  int cfunctionId;
-
-  U64 totals = 0;
-  U64 totalz = 0;
-  double totalc = 0.;
-  double totald = 0.;
-
-  /* Init */
-  if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1;
-  switch (cfunctionId)
-  {
-#ifdef COMPRESSOR0
-  case 0 : compP.compressionFunction = COMPRESSOR0; break;
-#endif
-#ifdef COMPRESSOR1
-  case 1 : compP.compressionFunction = COMPRESSOR1; break;
-#endif
-  default : compP.compressionFunction = DEFAULTCOMPRESSOR;
-  }
-
-  /* Loop for each file */
-  while (fileIdx<nbFiles) {
-      const char* inFileName = fileNamesTable[fileIdx++];
-      FILE* const inFile = fopen( inFileName, "rb" );
-      U64 const inFileSize = BMK_GetFileSize(inFileName);
-      size_t benchedSize = BMK_findMaxMem(inFileSize * 2) / 2;
-      unsigned nbChunks;
-      int maxCompressedChunkSize;
-      size_t readSize;
-      char* compressedBuffer;
-      struct chunkParameters* chunkP;
-      U32 crcOrig;
-
-      /* Check file and memory conditions */
-      if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
-      if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; }
-      if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; }
-      if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
-      if (benchedSize < inFileSize) {
-          DISPLAY("Not enough memory for '%s' full size; testing %u MB only...\n", inFileName, (unsigned)(benchedSize>>20));
-      }
-
-      /* Allocation */
-      nbChunks = (unsigned)(benchedSize / g_chunkSize) + 1;
-      chunkP = (struct chunkParameters*) malloc(nbChunks * sizeof(struct chunkParameters));
-      orig_buff = (char*)malloc(benchedSize);
-      maxCompressedChunkSize = LZ4_compressBound(g_chunkSize);
-      { size_t const compressedBuffSize = (size_t)(nbChunks * maxCompressedChunkSize);
-        compressedBuffer = (char*)malloc(compressedBuffSize); }
-
-      if (!orig_buff || !compressedBuffer){
-        DISPLAY("\nError: not enough memory!\n");
-        free(orig_buff);
-        free(compressedBuffer);
-        free(chunkP);
-        fclose(inFile);
-        return 12;
-      }
-
-      /* Init chunks data */
-      {   unsigned i;
-          size_t remaining = benchedSize;
-          char* in = orig_buff;
-          char* out = compressedBuffer;
-          for (i=0; i<nbChunks; i++) {
-              chunkP[i].id = i;
-              chunkP[i].origBuffer = in; in += g_chunkSize;
-              if (remaining > (size_t)g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
-              chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
-              chunkP[i].compressedSize = 0;
-      }   }
-
-      /* Fill input buffer */
-      DISPLAY("Loading %s...       \r", inFileName);
-      if (strlen(inFileName)>16) inFileName += strlen(inFileName)-16;   /* can only display 16 characters */
-      readSize = fread(orig_buff, 1, benchedSize, inFile);
-      fclose(inFile);
-
-      if (readSize != benchedSize) {
-        DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
-        free(orig_buff);
-        free(compressedBuffer);
-        free(chunkP);
-        return 13;
-      }
-
-      /* Calculating input Checksum */
-      crcOrig = XXH32(orig_buff, benchedSize,0);
-
-      /* Bench */
-      { int loopNb;
-        size_t cSize = 0;
-        double fastestC = 100000000., fastestD = 100000000.;
-        double ratio = 0.;
-        U32 crcCheck = 0;
-
-        DISPLAY("\r%79s\r", "");
-        for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
-          int nbLoops = 0;
-          clock_t clockStart, clockEnd;
-          unsigned chunkNb;
-
-          /* Compression */
-          DISPLAY("%2i#%1i-%-14.14s : %9i ->\r", cLevel, loopNb, inFileName, (int)benchedSize);
-          { size_t i; for (i=0; i<benchedSize; i++) compressedBuffer[i]=(char)i; }     /* warmimg up memory */
-
-          clockStart = clock();
-          while (clock() == clockStart);
-          clockStart = clock();
-          while (BMK_getClockSpan(clockStart) < TIMELOOP_CLOCK) {
-             for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
-                chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize, cLevel);
-             nbLoops++;
-          }
-          clockEnd = BMK_getClockSpan(clockStart);
-
-          nbLoops += !nbLoops;   /* avoid division by zero */
-          if ((double)clockEnd < fastestC*nbLoops) fastestC = (double)clockEnd/nbLoops;
-          cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
-          ratio = (double)cSize/(double)benchedSize*100.;
-
-          DISPLAY("%2i#%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r",
-                  cLevel, loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000);
-
-          /* Decompression */
-          { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; }     /* zeroing area, for CRC checking */
-
-          nbLoops = 0;
-          clockStart = clock();
-          while (clock() == clockStart);
-          clockStart = clock();
-          while (BMK_getClockSpan(clockStart) < TIMELOOP_CLOCK) {
-             for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
-                //chunkP[chunkNb].compressedSize = LZ4_decompress_fast(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].origSize);
-                LZ4_decompress_safe(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedSize, chunkP[chunkNb].origSize);
-             nbLoops++;
-          }
-          clockEnd = BMK_getClockSpan(clockStart);
-
-          nbLoops += !nbLoops;   /* avoid division by zero */
-          if ((double)clockEnd < fastestD*nbLoops) fastestD = (double)clockEnd/nbLoops;
-          DISPLAY("%2i#%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \r",
-                  cLevel, loopNb, inFileName, (int)benchedSize, (int)cSize, ratio,
-                  (double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000, (double)benchedSize / (fastestD / CLOCKS_PER_SEC) / 1000000 );
-
-          /* CRC Checking */
-          crcCheck = XXH32(orig_buff, benchedSize,0);
-          if (crcOrig!=crcCheck) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOrig, (unsigned)crcCheck); break; }
-        }
+    if (g_displayLevel == 1 && !g_additionalParam)
+        DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", LZ4_VERSION_STRING, LZ4_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
 
-        if (crcOrig==crcCheck) {
-            if (ratio < 100.)
-                DISPLAY("%2i#%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \n",
-                        cLevel, inFileName, (int)benchedSize, (int)cSize, ratio,
-                        (double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000, (double)benchedSize / (fastestD / CLOCKS_PER_SEC) / 1000000 );
-            else
-                DISPLAY("%2i#%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s  \n",
-                        cLevel, inFileName, (int)benchedSize, (int)cSize, ratio,
-                        (double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000, (double)benchedSize / (fastestD / CLOCKS_PER_SEC) / 1000000 );
-        }
-        totals += benchedSize;
-        totalz += cSize;
-        totalc += fastestC;
-        totald += fastestD;
-      }
+    if (cLevelLast < cLevel) cLevelLast = cLevel;
 
-      free(orig_buff);
-      free(compressedBuffer);
-      free(chunkP);
-  }
+    for (l=cLevel; l <= cLevelLast; l++) {
+        BMK_benchMem(srcBuffer, benchedSize,
+                     displayName, l,
+                     fileSizes, nbFiles);
+    }
+}
 
-  if (nbFiles > 1)
-        DISPLAY("%2i#%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", cLevel, "  TOTAL",
-                (long long unsigned)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100.,
-                (double)totals/(totalc/CLOCKS_PER_SEC)/1000000, (double)totals/(totald/CLOCKS_PER_SEC)/1000000);
 
-  if (BMK_pause) { DISPLAY("\npress enter...\n"); (void)getchar(); }
+/*! BMK_loadFiles() :
+    Loads `buffer` with content of files listed within `fileNamesTable`.
+    At most, fills `buffer` entirely */
+static void BMK_loadFiles(void* buffer, size_t bufferSize,
+                          size_t* fileSizes,
+                          const char** fileNamesTable, unsigned nbFiles)
+{
+    size_t pos = 0, totalSize = 0;
+    unsigned n;
+    for (n=0; n<nbFiles; n++) {
+        FILE* f;
+        U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
+        if (UTIL_isDirectory(fileNamesTable[n])) {
+            DISPLAYLEVEL(2, "Ignoring %s directory...       \n", fileNamesTable[n]);
+            fileSizes[n] = 0;
+            continue;
+        }
+        f = fopen(fileNamesTable[n], "rb");
+        if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
+        DISPLAYUPDATE(2, "Loading %s...       \r", fileNamesTable[n]);
+        if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n;   /* buffer too small - stop after this file */
+        { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
+          if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+          pos += readSize; }
+        fileSizes[n] = (size_t)fileSize;
+        totalSize += (size_t)fileSize;
+        fclose(f);
+    }
 
-  return 0;
+    if (totalSize == 0) EXM_THROW(12, "no data to bench");
 }
 
+static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
+                               int cLevel, int cLevelLast)
+{
+    void* srcBuffer;
+    size_t benchedSize;
+    size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
+    U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
+    char mfName[20] = {0};
+
+    if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
+
+    /* Memory allocation & restrictions */
+    benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
+    if (benchedSize==0) EXM_THROW(12, "not enough memory");
+    if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
+    if (benchedSize < totalSizeToLoad)
+        DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
+    srcBuffer = malloc(benchedSize);
+    if (!srcBuffer) EXM_THROW(12, "not enough memory");
+
+    /* Load input buffer */
+    BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
+
+    /* Bench */
+    snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
+    {   const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
+        BMK_benchCLevel(srcBuffer, benchedSize,
+                        displayName, cLevel, cLevelLast,
+                        fileSizes, nbFiles);
+    }
 
-int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel, int cLevelLast)
+    /* clean up */
+    free(srcBuffer);
+    free(fileSizes);
+}
+
+
+static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
 {
-    int i, res = 0;
+    char name[20] = {0};
+    size_t benchedSize = 10000000;
+    void* const srcBuffer = malloc(benchedSize);
 
-    if (cLevel > LZ4HC_MAX_CLEVEL) cLevel = LZ4HC_MAX_CLEVEL;
-    if (cLevelLast > LZ4HC_MAX_CLEVEL) cLevelLast = LZ4HC_MAX_CLEVEL;
-    if (cLevelLast < cLevel) cLevelLast = cLevel;
+    /* Memory allocation */
+    if (!srcBuffer) EXM_THROW(21, "not enough memory");
 
-    if (cLevelLast > cLevel) DISPLAY("Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
-    for (i=cLevel; i<=cLevelLast; i++) {
-        res = BMK_benchLevel(fileNamesTable, nbFiles, i);
-        if (res != 0) break;
-    }
+    /* Fill input buffer */
+    RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
+
+    /* Bench */
+    snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
+    BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1);
+
+    /* clean up */
+    free(srcBuffer);
+}
+
+
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
+                   int cLevel, int cLevelLast)
+{
+    double const compressibility = (double)g_compressibilityDefault / 100;
 
-    return res;
+    if (nbFiles == 0)
+        BMK_syntheticTest(cLevel, cLevelLast, compressibility);
+    else
+        BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast);
+    return 0;
 }
diff --git a/programs/bench.h b/programs/bench.h
index e84aaee..15def93 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -1,6 +1,6 @@
 /*
     bench.h - Demo program to benchmark open-source compression algorithm
-    Copyright (C) Yann Collet 2012-2015
+    Copyright (C) Yann Collet 2012-2016
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -20,15 +20,18 @@
     - LZ4 source repository : https://github.com/lz4/lz4
     - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
-#pragma once
+#ifndef BENCH_H_125623623633
+#define BENCH_H_125623623633
 
+#include <stddef.h>
 
-/* Main function */
-int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel, int cLevelLast);
-int BMK_benchLevel(const char** fileNamesTable, int nbFiles, int cLevel);
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
+                   int cLevel, int cLevelLast);
 
 /* Set Parameters */
-void BMK_setBlocksize(int bsize);
-void BMK_setNbIterations(int nbLoops);
-void BMK_setPause(void);
+void BMK_SetNbSeconds(unsigned nbLoops);
+void BMK_SetBlockSize(size_t blockSize);
+void BMK_setAdditionalParam(int additionalParam);
+void BMK_setNotificationLevel(unsigned level);
 
+#endif   /* BENCH_H_125623623633 */
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index f77be3d..e49bbcb 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -42,13 +42,6 @@
 /**************************************
 *  Compiler Options
 ***************************************/
-/* Disable some Visual warning messages */
-#ifdef _MSC_VER
-#  define _CRT_SECURE_NO_WARNINGS
-#  define _CRT_SECURE_NO_DEPRECATE     /* VS2005 */
-#  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
-#endif
-
 /* cf. http://man7.org/linux/man-pages/man7/feature_test_macros.7.html */
 #define _XOPEN_VERSION 600 /* POSIX.2001, for fileno() within <stdio.h> on unix */
 
@@ -56,6 +49,7 @@
 /****************************
 *  Includes
 *****************************/
+#include "util.h"     /* Compiler options, UTIL_HAS_CREATEFILELIST */
 #include <stdio.h>    /* fprintf, getchar */
 #include <stdlib.h>   /* exit, calloc, free */
 #include <string.h>   /* strcmp, strlen */
@@ -171,6 +165,9 @@ static int usage_advanced(void)
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
     DISPLAY( " -t     : test compressed file integrity\n");
     DISPLAY( " -m     : multiple input files (implies automatic output filenames)\n");
+#ifdef UTIL_HAS_CREATEFILELIST
+    DISPLAY( " -r     : operate recursively on directories (sets also -m)\n");
+#endif
     DISPLAY( " -l     : compress using Legacy format (Linux kernel compression)\n");
     DISPLAY( " -B#    : Block size [4-7](default : 7)\n");
     DISPLAY( " -BD    : Block dependency (improve compression ratio)\n");
@@ -179,10 +176,9 @@ static int usage_advanced(void)
     DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
     DISPLAY( "--[no-]sparse  : sparse mode (default:enabled on file, disabled on stdout)\n");
     DISPLAY( "Benchmark arguments :\n");
-    DISPLAY( "Benchmark arguments :\n");
     DISPLAY( " -b#    : benchmark file(s), using # compression level (default : 1) \n");
-    DISPLAY( " -e#    : test all compression levels from -bX to # (default: 1)\n");
-    DISPLAY( " -i#    : iteration loops [1-9](default : 3), benchmark mode only\n");
+    DISPLAY( " -e#    : test all compression levels from -bX to # (default : 1)\n");
+    DISPLAY( " -i#    : minimum evaluation time in seconds (default : 3s)\n");
 #if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
     DISPLAY( "Legacy arguments :\n");
     DISPLAY( " -c0    : fast compression\n");
@@ -304,6 +300,11 @@ int main(int argc, const char** argv)
     char nullOutput[] = NULL_OUTPUT;
     char extension[] = LZ4_EXTENSION;
     int  blockSize;
+#ifdef UTIL_HAS_CREATEFILELIST
+    const char** extendedFileList = NULL;
+    char* fileNamesBuf = NULL;
+    unsigned fileNamesNb, recursive=0;
+#endif
 
     /* Init */
     programName = argv[0];
@@ -419,7 +420,7 @@ int main(int argc, const char** argv)
                         case '7':
                             {   int B = argument[1] - '0';
                                 blockSize = LZ4IO_setBlockSizeID(B);
-                                BMK_setBlocksize(blockSize);
+                                BMK_SetBlockSize(blockSize);
                                 argument++;
                                 break;
                             }
@@ -437,24 +438,28 @@ int main(int argc, const char** argv)
                         inFileNames = (const char**) malloc(argc * sizeof(char*));
                     break;
 
+#ifdef UTIL_HAS_CREATEFILELIST
+                        /* recursive */
+                case 'r': recursive=1;  /* without break */
+#endif
                     /* Treat non-option args as input files.  See https://code.google.com/p/lz4/issues/detail?id=151 */
                 case 'm': multiple_inputs=1;
                     if (inFileNames == NULL)
                         inFileNames = (const char**) malloc(argc * sizeof(char*));
                     break;
 
-                    /* Modify Nb Iterations (benchmark only) */
+                    /* Modify Nb Seconds (benchmark only) */
                 case 'i':
                     {   unsigned iters;
                         argument++;
                         iters = readU32FromChar(&argument);
                         argument--;
-                        BMK_setNbIterations(iters);
+                        BMK_SetNbSeconds(iters);
                     }
                     break;
 
                     /* Pause at the end (hidden option) */
-                case 'p': main_pause=1; BMK_setPause(); break;
+                case 'p': main_pause=1; break;
 
                     /* Specific commands for customized versions */
                 EXTENDED_ARGUMENTS;
@@ -487,7 +492,22 @@ int main(int argc, const char** argv)
     if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10);
 
     /* No input filename ==> use stdin */
-    if (multiple_inputs) input_filename = inFileNames[0], output_filename = (const char*)(inFileNames[0]);
+    if (multiple_inputs) {
+        input_filename = inFileNames[0];
+        output_filename = (const char*)(inFileNames[0]);
+#ifdef UTIL_HAS_CREATEFILELIST
+        if (recursive) {  /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
+            extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb);
+            if (extendedFileList) {
+                unsigned u;
+                for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
+                free((void*)inFileNames);
+                inFileNames = extendedFileList;
+                ifnIdx = fileNamesNb;
+            }
+        }
+#endif
+    }
     if(!input_filename) { input_filename=stdinmark; }
 
     /* Check if input is defined as console; trigger an error in this case */
@@ -498,9 +518,8 @@ int main(int argc, const char** argv)
 
     /* Check if benchmark is selected */
     if (bench) {
-        int bmkResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel, cLevelLast);
-        free((void*)inFileNames);
-        return bmkResult;
+        operationResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel, cLevelLast);
+        goto _cleanup;
     }
 
     /* No output filename ==> try to select one automatically (when possible) */
@@ -566,7 +585,12 @@ int main(int argc, const char** argv)
 
 _cleanup:
     if (main_pause) waitEnter();
-    free(dynNameSpace);
-    free((void*)inFileNames);
+    if (dynNameSpace) free(dynNameSpace);
+#ifdef UTIL_HAS_CREATEFILELIST
+    if (extendedFileList)
+        UTIL_freeFileList(extendedFileList, fileNamesBuf);
+    else
+#endif
+        free((void*)inFileNames);
     return operationResult;
 }
diff --git a/programs/util.h b/programs/util.h
new file mode 100644
index 0000000..d76ce6a
--- /dev/null
+++ b/programs/util.h
@@ -0,0 +1,503 @@
+/*
+    util.h - utility functions
+    Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    You can contact the author at :
+    - LZ4 source repository : https://github.com/lz4/lz4
+    - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+#ifndef UTIL_H_MODULE
+#define UTIL_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* **************************************
+*  Compiler Options
+****************************************/
+#if defined(__INTEL_COMPILER)
+#  pragma warning(disable : 177)    /* disable: message #177: function was declared but never referenced */
+#endif
+#if defined(_MSC_VER)
+#  define _CRT_SECURE_NO_WARNINGS    /* Disable some Visual warning messages for fopen, strncpy */
+#  define _CRT_SECURE_NO_DEPRECATE   /* VS2005 */
+#  pragma warning(disable : 4127)    /* disable: C4127: conditional expression is constant */
+#if _MSC_VER <= 1800                 /* (1800 = Visual Studio 2013) */
+    #define snprintf sprintf_s       /* snprintf unsupported by Visual <= 2013 */
+#endif
+#endif
+
+
+/* Unix Large Files support (>4GB) */
+#if !defined(__LP64__)                                  /* No point defining Large file for 64 bit */
+#   define _FILE_OFFSET_BITS 64                         /* turn off_t into a 64-bit type for ftello, fseeko */
+#   if defined(__sun__) && !defined(_LARGEFILE_SOURCE)  /* Sun Solaris 32-bits requires specific definitions */
+#      define _LARGEFILE_SOURCE                         /* fseeko, ftello */
+#   elif !defined(_LARGEFILE64_SOURCE)
+#      define _LARGEFILE64_SOURCE                       /* off64_t, fseeko64, ftello64 */
+#   endif
+#endif
+
+
+/*-****************************************
+*  Dependencies
+******************************************/
+#include <stdlib.h>     /* features.h with _POSIX_C_SOURCE, malloc */
+#include <stdio.h>      /* fprintf */
+#include <string.h>     /* strerr, strlen, memcpy */
+#include <stddef.h>     /* ptrdiff_t */
+#include <sys/types.h>  /* stat, utime */
+#include <sys/stat.h>   /* stat */
+#if defined(_MSC_VER)
+	#include <sys/utime.h>   /* utime */
+	#include <io.h>          /* _chmod */
+#else
+	#include <unistd.h>     /* chown, stat */
+	#include <utime.h>      /* utime */
+#endif
+#include <time.h>       /* time */
+#include <errno.h>
+
+
+/* *************************************
+*  Constants
+***************************************/
+#define LIST_SIZE_INCREASE   (8*1024)
+
+
+/*-****************************************
+*  Compiler specifics
+******************************************/
+#if defined(__GNUC__)
+#  define UTIL_STATIC static __attribute__((unused))
+#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#  define UTIL_STATIC static inline
+#elif defined(_MSC_VER)
+#  define UTIL_STATIC static __inline
+#else
+#  define UTIL_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
+#endif
+
+
+/*-****************************************
+*  Sleep functions: Windows - Posix - others
+******************************************/
+#if defined(_WIN32)
+#  include <windows.h>
+#  define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
+#  define UTIL_sleep(s) Sleep(1000*s)
+#  define UTIL_sleepMilli(milli) Sleep(milli)
+#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__)))
+#  include <unistd.h>
+#  include <sys/resource.h> /* setpriority */
+#  include <time.h>         /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
+#  if defined(PRIO_PROCESS)
+#    define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
+#  else
+#    define SET_HIGH_PRIORITY /* disabled */
+#  endif
+#  define UTIL_sleep(s) sleep(s)
+#  if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L)
+#      define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
+#  else
+#      define UTIL_sleepMilli(milli) /* disabled */
+#  endif
+#else
+#  define SET_HIGH_PRIORITY      /* disabled */
+#  define UTIL_sleep(s)          /* disabled */
+#  define UTIL_sleepMilli(milli) /* disabled */
+#endif
+
+
+/*-**************************************************************
+*  Basic Types
+*****************************************************************/
+#if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+  typedef  uint8_t BYTE;
+  typedef uint16_t U16;
+  typedef  int16_t S16;
+  typedef uint32_t U32;
+  typedef  int32_t S32;
+  typedef uint64_t U64;
+  typedef  int64_t S64;
+#else
+  typedef unsigned char       BYTE;
+  typedef unsigned short      U16;
+  typedef   signed short      S16;
+  typedef unsigned int        U32;
+  typedef   signed int        S32;
+  typedef unsigned long long  U64;
+  typedef   signed long long  S64;
+#endif 
+
+
+/*-****************************************
+*  Time functions
+******************************************/
+#if !defined(_WIN32)
+   typedef clock_t UTIL_time_t;
+   UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; }
+   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); }
+   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+#else
+   typedef LARGE_INTEGER UTIL_time_t;
+   UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); }
+   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); }
+   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
+   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
+#endif
+
+
+/* returns time span in microseconds */
+UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond )
+{
+    UTIL_time_t clockEnd;
+    UTIL_getTime(&clockEnd);
+    return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd);
+}
+
+
+UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
+{
+    UTIL_time_t clockStart, clockEnd;
+    UTIL_getTime(&clockStart);
+    do {
+        UTIL_getTime(&clockEnd);
+    } while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
+}
+
+
+
+/*-****************************************
+*  File functions
+******************************************/
+#if defined(_MSC_VER)
+	#define chmod _chmod
+	typedef struct _stat64 stat_t;
+#else
+    typedef struct stat stat_t;
+#endif
+
+
+UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
+{
+    int res = 0;
+    struct utimbuf timebuf;
+
+	timebuf.actime = time(NULL);
+	timebuf.modtime = statbuf->st_mtime;
+	res += utime(filename, &timebuf);  /* set access and modification times */
+
+#if !defined(_WIN32)
+    res += chown(filename, statbuf->st_uid, statbuf->st_gid);  /* Copy ownership */
+#endif
+
+    res += chmod(filename, statbuf->st_mode & 07777);  /* Copy file permissions */
+
+    errno = 0;
+    return -res; /* number of errors is returned */
+}
+
+
+UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
+{
+    int r;
+#if defined(_MSC_VER)
+    r = _stat64(infilename, statbuf);
+    if (r || !(statbuf->st_mode & S_IFREG)) return 0;   /* No good... */
+#else
+    r = stat(infilename, statbuf);
+    if (r || !S_ISREG(statbuf->st_mode)) return 0;   /* No good... */
+#endif
+    return 1;
+}
+
+
+UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
+{
+    int r;
+#if defined(_MSC_VER)
+    struct _stat64 statbuf;
+    r = _stat64(infilename, &statbuf);
+    if (r || !(statbuf.st_mode & S_IFREG)) return 0;   /* No good... */
+#else
+    struct stat statbuf;
+    r = stat(infilename, &statbuf);
+    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
+#endif
+    return (U64)statbuf.st_size;
+}
+
+
+UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
+{
+    U64 total = 0;
+    unsigned n;
+    for (n=0; n<nbFiles; n++)
+        total += UTIL_getFileSize(fileNamesTable[n]);
+    return total;
+}
+
+
+UTIL_STATIC int UTIL_doesFileExists(const char* infilename)
+{
+    int r;
+#if defined(_MSC_VER)
+    struct _stat64 statbuf;
+    r = _stat64(infilename, &statbuf);
+    if (r || !(statbuf.st_mode & S_IFREG)) return 0;   /* No good... */
+#else
+    struct stat statbuf;
+    r = stat(infilename, &statbuf);
+    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
+#endif
+    return 1;
+}
+
+
+UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
+{
+    int r;
+#if defined(_MSC_VER)
+    struct _stat64 statbuf;
+    r = _stat64(infilename, &statbuf);
+    if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
+#else
+    struct stat statbuf;
+    r = stat(infilename, &statbuf);
+    if (!r && S_ISDIR(statbuf.st_mode)) return 1;
+#endif
+    return 0;
+}
+
+/*
+ * A modified version of realloc().
+ * If UTIL_realloc() fails the original block is freed.
+*/
+UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
+{
+    void *newptr = realloc(ptr, size);
+    if (newptr) return newptr;
+    free(ptr);
+    return NULL;
+}
+
+
+#ifdef _WIN32
+#  define UTIL_HAS_CREATEFILELIST
+
+UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
+{
+    char* path;
+    int dirLength, fnameLength, pathLength, nbFiles = 0;
+    WIN32_FIND_DATA cFile;
+    HANDLE hFile;
+
+    dirLength = (int)strlen(dirName);
+    path = (char*) malloc(dirLength + 3);
+    if (!path) return 0;
+
+    memcpy(path, dirName, dirLength);
+    path[dirLength] = '\\';
+    path[dirLength+1] = '*';
+    path[dirLength+2] = 0;
+
+    hFile=FindFirstFile(path, &cFile);
+    if (hFile == INVALID_HANDLE_VALUE) {
+        fprintf(stderr, "Cannot open directory '%s'\n", dirName);
+        return 0;
+    }
+    free(path);
+
+    do {
+        fnameLength = (int)strlen(cFile.cFileName);
+        path = (char*) malloc(dirLength + fnameLength + 2);
+        if (!path) { FindClose(hFile); return 0; }
+        memcpy(path, dirName, dirLength);
+        path[dirLength] = '\\';
+        memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
+        pathLength = dirLength+1+fnameLength;
+        path[pathLength] = 0;
+        if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+            if (strcmp (cFile.cFileName, "..") == 0 ||
+                strcmp (cFile.cFileName, ".") == 0) continue;
+
+            nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd);  /* Recursively call "UTIL_prepareFileList" with the new path. */
+            if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
+        }
+        else if ((cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED)) {
+            if (*bufStart + *pos + pathLength >= *bufEnd) {
+                ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
+                *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
+                *bufEnd = *bufStart + newListSize;
+                if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
+            }
+            if (*bufStart + *pos + pathLength < *bufEnd) {
+                strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
+                *pos += pathLength + 1;
+                nbFiles++;
+            }
+        }
+        free(path);
+    } while (FindNextFile(hFile, &cFile));
+
+    FindClose(hFile);
+    return nbFiles;
+}
+
+#elif (defined(__APPLE__) && defined(__MACH__)) || \
+     ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */
+#  define UTIL_HAS_CREATEFILELIST
+#  include <dirent.h>       /* opendir, readdir */
+
+UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
+{
+    DIR *dir;
+    struct dirent *entry;
+    char* path;
+    int dirLength, fnameLength, pathLength, nbFiles = 0;
+
+    if (!(dir = opendir(dirName))) {
+        fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
+        return 0;
+    }
+
+    dirLength = (int)strlen(dirName);
+    errno = 0;
+    while ((entry = readdir(dir)) != NULL) {
+        if (strcmp (entry->d_name, "..") == 0 ||
+            strcmp (entry->d_name, ".") == 0) continue;
+        fnameLength = (int)strlen(entry->d_name);
+        path = (char*) malloc(dirLength + fnameLength + 2);
+        if (!path) { closedir(dir); return 0; }
+        memcpy(path, dirName, dirLength);
+        path[dirLength] = '/';
+        memcpy(path+dirLength+1, entry->d_name, fnameLength);
+        pathLength = dirLength+1+fnameLength;
+        path[pathLength] = 0;
+
+        if (UTIL_isDirectory(path)) {
+            nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd);  /* Recursively call "UTIL_prepareFileList" with the new path. */
+            if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
+        } else {
+            if (*bufStart + *pos + pathLength >= *bufEnd) {
+                ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
+                *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
+                *bufEnd = *bufStart + newListSize;
+                if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
+            }
+            if (*bufStart + *pos + pathLength < *bufEnd) {
+                strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
+                *pos += pathLength + 1;
+                nbFiles++;
+            }
+        }
+        free(path);
+        errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
+    }
+
+    if (errno != 0) {
+        fprintf(stderr, "readdir(%s) error: %s\n", dirName, strerror(errno));
+        free(*bufStart);
+        *bufStart = NULL;
+    }
+    closedir(dir);
+    return nbFiles;
+}
+
+#else
+
+UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
+{
+    (void)bufStart; (void)bufEnd; (void)pos;
+    fprintf(stderr, "Directory %s ignored (zstd compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
+    return 0;
+}
+
+#endif /* #ifdef _WIN32 */
+
+/*
+ * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
+ *                       and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
+ * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
+ * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
+ */
+UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb)
+{
+    size_t pos;
+    unsigned i, nbFiles;
+    char *bufend, *buf;
+    const char** fileTable;
+
+    buf = (char*)malloc(LIST_SIZE_INCREASE);
+    if (!buf) return NULL;
+    bufend = buf + LIST_SIZE_INCREASE;
+
+    for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
+        if (!UTIL_isDirectory(inputNames[i])) {
+            size_t len = strlen(inputNames[i]);
+            if (buf + pos + len >= bufend) {
+                ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
+                buf = (char*)UTIL_realloc(buf, newListSize);
+                bufend = buf + newListSize;
+                if (!buf) return NULL;
+            }
+            if (buf + pos + len < bufend) {
+                strncpy(buf + pos, inputNames[i], bufend - (buf + pos));
+                pos += len + 1;
+                nbFiles++;
+            }
+        } else {
+            nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend);
+            if (buf == NULL) return NULL;
+    }   }
+
+    if (nbFiles == 0) { free(buf); return NULL; }
+
+    fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
+    if (!fileTable) { free(buf); return NULL; }
+
+    for (i=0, pos=0; i<nbFiles; i++)
+    {
+        fileTable[i] = buf + pos;
+        pos += strlen(fileTable[i]) + 1;
+    }
+
+    if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
+
+    *allocatedBuffer = buf;
+    *allocatedNamesNb = nbFiles;
+
+    return fileTable;
+}
+
+
+UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
+{
+    if (allocatedBuffer) free(allocatedBuffer);
+    if (filenameTable) free((void*)filenameTable);
+}
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* UTIL_H_MODULE */
diff --git a/tests/Makefile b/tests/Makefile
index bfbafd8..0dd8a59 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -33,6 +33,7 @@
 
 DESTDIR ?=
 PREFIX  ?= /usr/local
+VOID    := /dev/null
 BINDIR  := $(PREFIX)/bin
 MANDIR  := $(PREFIX)/share/man/man1
 LZ4DIR  := ../lib
@@ -50,10 +51,8 @@ FLAGS   := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
 EXT =.exe
-VOID = nul
 else
 EXT =
-VOID = /dev/null
 endif