Merge pull request #6 from Cyan4973/streaming

Streaming

13 files changed, 1020 insertions, 448 deletions

diff --git a/.travis.yml b/.travis.yml
index 2880427..472ca18 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,7 @@ script: make test
 before_install:
   - sudo apt-get update -qq
   - sudo apt-get install -qq gcc-multilib
+  - sudo apt-get install -qq valgrind
 
 env:
   - LZ4_TRAVIS_CI_ENV=-m32
diff --git a/Makefile b/Makefile
index 11fa7b3..ede6844 100644
--- a/Makefile
+++ b/Makefile
@@ -92,7 +92,7 @@ lz4programs: lz4.c lz4hc.c
 liblz4: lz4.c lz4hc.c
 	@echo compiling static library
 	@$(CC) $(CFLAGS) -c $^
-	@ar rcs liblz4.a lz4.o lz4hc.o
+	@$(AR) rcs liblz4.a lz4.o lz4hc.o
 	@echo compiling dynamic library
 	@$(CC) $(CFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
 	@echo creating versioned links
@@ -105,25 +105,25 @@ clean:
 	@echo Cleaning completed
 
 
-#make install option is reserved to Linux & OSX targets
+#make install option is designed for Linux & OSX targets only
 ifneq (,$(filter $(shell uname),Linux Darwin))
 
 install: liblz4
 	@install -d -m 755 $(DESTDIR)$(LIBDIR)/ $(DESTDIR)$(INCLUDEDIR)/
-	@install -m 755 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
 	@install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
 	@cp -a liblz4.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)
 	@cp -a liblz4.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
-	@install -m 755 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
-	@install -m 755 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
+	@install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
+	@install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
+	@install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
 	@echo lz4 static and shared library installed
 	@cd $(PRGDIR); $(MAKE) -e install
 
 uninstall:
-	[ -x $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a
 	rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
 	rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
 	[ -x $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
+	[ -f $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a
 	[ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4.h
 	[ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
 	@echo lz4 libraries successfully uninstalled
diff --git a/lz4.c b/lz4.c
index 8001edb..e1bcb25 100644..100755
--- a/lz4.c
+++ b/lz4.c
@@ -35,15 +35,6 @@
    Tuning parameters
 **************************************/
 /*
- * MEMORY_USAGE :
- * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
- * Increasing memory usage improves compression ratio
- * Reduced memory usage can improve speed, due to cache effect
- * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
- */
-#define MEMORY_USAGE 14
-
-/*
  * HEAPMODE :
  * Select how default compression functions will allocate memory for their hash table,
  * in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)).
@@ -56,8 +47,9 @@
 **************************************/
 /* 32 or 64 bits ? */
 #if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
-  || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
-  || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
+  || defined(__powerpc64__) || defined(__powerpc64le__) \
+  || defined(__ppc64__) || defined(__ppc64le__) \
+  || defined(__PPC64__) || defined(__PPC64LE__) \
   || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) )   /* Detects 64 bits mode */
 #  define LZ4_ARCH64 1
 #else
@@ -68,6 +60,7 @@
  * Little Endian or Big Endian ?
  * Overwrite the #define below if you know your architecture endianess
  */
+#include <stdlib.h>   /* Apparently required to detect endianess */
 #if defined (__GLIBC__)
 #  include <endian.h>
 #  if (__BYTE_ORDER == __BIG_ENDIAN)
@@ -118,7 +111,6 @@
 #endif
 
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  include <intrin.h>                    /* For Visual 2005 */
 #  if LZ4_ARCH64   /* 64-bits */
 #    pragma intrinsic(_BitScanForward64) /* For Visual 2005 */
@@ -128,15 +120,6 @@
 #    pragma intrinsic(_BitScanReverse)   /* For Visual 2005 */
 #  endif
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
-#else
-#  ifdef __GNUC__
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  else
-#    define FORCE_INLINE static inline
-#  endif
-#endif
-
-#ifdef _MSC_VER  /* Visual Studio */
 #  define lz4_bswap16(x) _byteswap_ushort(x)
 #else
 #  define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))
@@ -224,9 +207,9 @@ typedef struct {size_t v;} _PACKED size_t_S;
 /**************************************
    Constants
 **************************************/
-#define LZ4_HASHLOG   (MEMORY_USAGE-2)
-#define HASHTABLESIZE (1 << MEMORY_USAGE)
-#define HASHNBCELLS4  (1 << LZ4_HASHLOG)
+#define LZ4_HASHLOG   (LZ4_MEMORY_USAGE-2)
+#define HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
+#define HASH_SIZE_U32 (1 << LZ4_HASHLOG)
 
 #define MINMATCH 4
 
@@ -255,16 +238,18 @@ static const int LZ4_minLength = (MFLIMIT+1);
    Structures and local types
 **************************************/
 typedef struct {
-    U32 hashTable[HASHNBCELLS4];
+    U32  hashTable[HASH_SIZE_U32];
+    U32  currentOffset;
+    U32  initCheck;
+    const BYTE* dictionary;
     const BYTE* bufferStart;
-    const BYTE* base;
-    const BYTE* nextBlock;
-} LZ4_Data_Structure;
+    U32  dictSize;
+} LZ4_dict_t_internal;
 
-typedef enum { notLimited = 0, limited = 1 } limitedOutput_directive;
+typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
 typedef enum { byPtr, byU32, byU16 } tableType_t;
 
-typedef enum { noDict = 0, withPrefix64k = 1, withExtDict=2 } dict_directive;
+typedef enum { noDict = 0, withPrefix64k = 1, usingExtDict = 2 } dict_directive;
 
 typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
 typedef enum { full = 0, partial = 1 } earlyEnd_directive;
@@ -289,12 +274,12 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive;
 /**************************************
    Macros
 **************************************/
+#define LZ4_STATIC_ASSERT(c)    { enum { LZ4_static_assert = 1/(!!(c)) }; }   /* use only *after* variable declarations */
 #if LZ4_ARCH64 || !defined(__GNUC__)
-#  define LZ4_WILDCOPY(d,s,e)     { do { LZ4_COPY8(d,s) } while (d<e); }           /* at the end, d>=e; */
+#  define LZ4_WILDCOPY(d,s,e)   { do { LZ4_COPY8(d,s) } while (d<e); }        /* at the end, d>=e; */
 #else
-#  define LZ4_WILDCOPY(d,s,e)     { if (likely(e-d <= 8)) LZ4_COPY8(d,s) else do { LZ4_COPY8(d,s) } while (d<e); }
+#  define LZ4_WILDCOPY(d,s,e)   { if (likely(e-d <= 8)) LZ4_COPY8(d,s) else do { LZ4_COPY8(d,s) } while (d<e); }
 #endif
-#define LZ4_SECURECOPY(d,s,e)     { if (d<e) LZ4_WILDCOPY(d,s,e); }
 
 
 /****************************
@@ -302,7 +287,7 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive;
 ****************************/
 #if LZ4_ARCH64
 
-FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)
+int LZ4_NbCommonBytes (register U64 val)
 {
 # if defined(LZ4_BIG_ENDIAN)
 #   if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
@@ -334,7 +319,7 @@ FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)
 
 #else
 
-FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
+int LZ4_NbCommonBytes (register U32 val)
 {
 # if defined(LZ4_BIG_ENDIAN)
 #   if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
@@ -371,7 +356,7 @@ FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
 ****************************/
 int LZ4_compressBound(int isize)  { return LZ4_COMPRESSBOUND(isize); }
 
-FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType)
+static int LZ4_hashSequence(U32 sequence, tableType_t tableType)
 {
     if (tableType == byU16)
         return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
@@ -379,9 +364,9 @@ FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType)
         return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
 }
 
-FORCE_INLINE int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); }
+static int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); }
 
-FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     switch (tableType)
     {
@@ -391,58 +376,96 @@ FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, t
     }
 }
 
-FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     U32 h = LZ4_hashPosition(p, tableType);
     LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
 }
 
-FORCE_INLINE const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
     if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
     { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
 }
 
-FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     U32 h = LZ4_hashPosition(p, tableType);
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
 }
 
+static unsigned LZ4_count(const BYTE* pIn, const BYTE* pRef, const BYTE* pInLimit)
+{
+    const BYTE* const pStart = pIn;
+
+    while (likely(pIn<pInLimit-(STEPSIZE-1)))
+    {
+        size_t diff = AARCH(pRef) ^ AARCH(pIn);
+        if (!diff) { pIn+=STEPSIZE; pRef+=STEPSIZE; continue; }
+        pIn += LZ4_NbCommonBytes(diff);
+        return (unsigned)(pIn - pStart);
+    }
+    if (LZ4_ARCH64) if ((pIn<(pInLimit-3)) && (A32(pRef) == A32(pIn))) { pIn+=4; pRef+=4; }
+    if ((pIn<(pInLimit-1)) && (A16(pRef) == A16(pIn))) { pIn+=2; pRef+=2; }
+    if ((pIn<pInLimit) && (*pRef == *pIn)) pIn++;
+
+    return (unsigned)(pIn - pStart);
+}
+
 
-FORCE_INLINE int LZ4_compress_generic(
+static int LZ4_compress_generic(
                  void* ctx,
                  const char* source,
                  char* dest,
                  int inputSize,
                  int maxOutputSize,
 
-                 limitedOutput_directive limitedOutput,
+                 limitedOutput_directive outputLimited,
                  tableType_t tableType,
                  dict_directive dict)
 {
+    LZ4_dict_t_internal* const dictPtr = (LZ4_dict_t_internal*)ctx;
+
     const BYTE* ip = (const BYTE*) source;
-    const BYTE* const base = (dict==withPrefix64k) ? ((LZ4_Data_Structure*)ctx)->base : (const BYTE*) source;
-    const BYTE* const lowLimit = ((dict==withPrefix64k) ? ((LZ4_Data_Structure*)ctx)->bufferStart : (const BYTE*)source);
+    const BYTE* base;
+    const BYTE* lowLimit;
+    const BYTE* const dictionary = dictPtr->dictionary;
+    const BYTE* const dictEnd = dictionary + dictPtr->dictSize;
+    const size_t dictDelta = dictEnd - (const BYTE*)source;
     const BYTE* anchor = (const BYTE*) source;
     const BYTE* const iend = ip + inputSize;
     const BYTE* const mflimit = iend - MFLIMIT;
     const BYTE* const matchlimit = iend - LASTLITERALS;
 
     BYTE* op = (BYTE*) dest;
-    BYTE* const oend = op + maxOutputSize;
+    BYTE* const olimit = op + maxOutputSize;
 
-    int length;
     const int skipStrength = SKIPSTRENGTH;
     U32 forwardH;
+    size_t refDelta=0;
 
     /* Init conditions */
-    if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;                                /* Unsupported input size, too large (or negative) */
-    if ((dict==withPrefix64k) && (ip != ((LZ4_Data_Structure*)ctx)->nextBlock)) return 0;   /* must continue from end of previous block */
-    if (dict==withPrefix64k) ((LZ4_Data_Structure*)ctx)->nextBlock=iend;                    /* do it now, due to potential early exit */
-    if ((tableType == byU16) && (inputSize>=(int)LZ4_64KLIMIT)) return 0;                  /* Size too large (not within 64K limit) */
-    if (inputSize<LZ4_minLength) goto _last_literals;                                      /* Input too small, no compression (all literals) */
+    if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;          /* Unsupported input size, too large (or negative) */
+    switch(dict)
+    {
+    case noDict:
+    default:
+        base = (const BYTE*)source;
+        lowLimit = (const BYTE*)source;
+        break;
+    case withPrefix64k:
+        base = (const BYTE*)source - dictPtr->currentOffset;
+        lowLimit = (const BYTE*)source - dictPtr->dictSize;
+        if (lowLimit < base) lowLimit = base;
+        break;
+    case usingExtDict:
+        base = (const BYTE*)source - dictPtr->currentOffset;
+        lowLimit = (const BYTE*)source;
+        break;
+    }
+    if ((tableType == byU16) && (inputSize>=(int)LZ4_64KLIMIT)) return 0;   /* Size too large (not within 64K limit) */
+    if (inputSize<LZ4_minLength) goto _last_literals;                       /* Input too small, no compression (all literals) */
 
     /* First Byte */
     LZ4_putPosition(ip, ctx, tableType, base);
@@ -451,98 +474,139 @@ FORCE_INLINE int LZ4_compress_generic(
     /* Main Loop */
     for ( ; ; )
     {
-        int findMatchAttempts = (1U << skipStrength) + 3;
         const BYTE* forwardIp = ip;
         const BYTE* ref;
         BYTE* token;
+        {
+            int step=1;
+            int searchMatchNb = (1U << skipStrength) + 3;
 
-        /* Find a match */
-        do {
-            U32 h = forwardH;
-            int step = findMatchAttempts++ >> skipStrength;
-            ip = forwardIp;
-            forwardIp = ip + step;
+            /* Find a match */
+            do {
+                U32 h = forwardH;
+                ip = forwardIp;
+                forwardIp += step;
+                step = searchMatchNb++ >> skipStrength;
+                if (unlikely (step>8)) step=8;   // slows down uncompressible data; required for valid forwardIp
 
-            if (unlikely(forwardIp > mflimit)) { goto _last_literals; }
+                if (unlikely(ip > mflimit)) goto _last_literals;
 
-            forwardH = LZ4_hashPosition(forwardIp, tableType);
-            ref = LZ4_getPositionOnHash(h, ctx, tableType, base);
-            LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
+                ref = LZ4_getPositionOnHash(h, ctx, tableType, base);
+                if (dict==usingExtDict)
+                {
+                    if (ref<(const BYTE*)source)
+                    {
+                        refDelta = dictDelta;
+                        lowLimit = dictionary;
+                    }
+                    else
+                    {
+                        refDelta = 0;
+                        lowLimit = (const BYTE*)source;
+                    }
+                }
+                forwardH = LZ4_hashPosition(forwardIp, tableType);
+                LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
 
-        } while ((ref + MAX_DISTANCE < ip) || (A32(ref) != A32(ip)));
+            } while ( ((tableType==byU16)? 0 : (ref + MAX_DISTANCE < ip)) || (A32(ref+refDelta) != A32(ip)) );
+        }
 
         /* Catch up */
-        while ((ip>anchor) && (ref > lowLimit) && (unlikely(ip[-1]==ref[-1]))) { ip--; ref--; }
+        while ((ip>anchor) && (ref+refDelta > lowLimit) && (unlikely(ip[-1]==ref[refDelta-1]))) { ip--; ref--; }
 
-        /* Encode Literal length */
-        length = (int)(ip - anchor);
-        token = op++;
-        if ((limitedOutput) && (unlikely(op + length + (2 + 1 + LASTLITERALS) + (length/255) > oend))) return 0;   /* Check output limit */
-        if (length>=(int)RUN_MASK)
         {
-            int len = length-RUN_MASK;
-            *token=(RUN_MASK<<ML_BITS);
-            for(; len >= 255 ; len-=255) *op++ = 255;
-            *op++ = (BYTE)len;
-        }
-        else *token = (BYTE)(length<<ML_BITS);
+            /* Encode Literal length */
+            unsigned litLength = (unsigned)(ip - anchor);
+            token = op++;
+            if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) return 0;   /* Check output limit */
+            if (litLength>=RUN_MASK)
+            {
+                int len = (int)litLength-RUN_MASK;
+                *token=(RUN_MASK<<ML_BITS);
+                for(; len >= 255 ; len-=255) *op++ = 255;
+                *op++ = (BYTE)len;
+            }
+            else *token = (BYTE)(litLength<<ML_BITS);
 
-        /* Copy Literals */
-        { BYTE* end=(op)+(length); LZ4_WILDCOPY(op,anchor,end); op=end; }
+            /* Copy Literals */
+            { BYTE* end=(op)+(litLength); LZ4_WILDCOPY(op,anchor,end); op=end; }
+        }
 
 _next_match:
         /* Encode Offset */
         LZ4_WRITE_LITTLEENDIAN_16(op,(U16)(ip-ref));
 
-        /* Start Counting */
-        ip+=MINMATCH; ref+=MINMATCH;    /* MinMatch already verified */
-        anchor = ip;
-        while (likely(ip<matchlimit-(STEPSIZE-1)))
-        {
-            size_t diff = AARCH(ref) ^ AARCH(ip);
-            if (!diff) { ip+=STEPSIZE; ref+=STEPSIZE; continue; }
-            ip += LZ4_NbCommonBytes(diff);
-            goto _endCount;
-        }
-        if (LZ4_ARCH64) if ((ip<(matchlimit-3)) && (A32(ref) == A32(ip))) { ip+=4; ref+=4; }
-        if ((ip<(matchlimit-1)) && (A16(ref) == A16(ip))) { ip+=2; ref+=2; }
-        if ((ip<matchlimit) && (*ref == *ip)) ip++;
-_endCount:
-
         /* Encode MatchLength */
-        length = (int)(ip - anchor);
-        if ((limitedOutput) && (unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend))) return 0;    /* Check output limit */
-        if (length>=(int)ML_MASK)
+        ref += refDelta;
         {
-            *token += ML_MASK;
-            length -= ML_MASK;
-            for (; length > 509 ; length-=510) { *op++ = 255; *op++ = 255; }
-            if (length >= 255) { length-=255; *op++ = 255; }
-            *op++ = (BYTE)length;
+            unsigned matchLength;
+
+            if ((dict==usingExtDict) && (lowLimit==dictionary))
+            {
+                const BYTE* limit = ip + (dictEnd-ref);
+                if (limit > matchlimit) limit = matchlimit;
+                matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, limit);
+                ip += MINMATCH + matchLength;
+                if (ip==limit)
+                {
+                    unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit);
+                    matchLength += more;
+                    ip += more;
+                }
+            }
+            else
+            {
+                matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit);
+                ip += MINMATCH + matchLength;
+            }
+
+            if (matchLength>=ML_MASK)
+            {
+                if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit))) return 0;    /* Check output limit */
+                *token += ML_MASK;
+                matchLength -= ML_MASK;
+                for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; }
+                if (matchLength >= 255) { matchLength-=255; *op++ = 255; }
+                *op++ = (BYTE)matchLength;
+            }
+            else *token += (BYTE)(matchLength);
         }
-        else *token += (BYTE)(length);
+
+        anchor = ip;
 
         /* Test end of chunk */
-        if (ip > mflimit) { anchor = ip;  break; }
+        if (ip > mflimit) break;
 
         /* Fill table */
         LZ4_putPosition(ip-2, ctx, tableType, base);
 
         /* Test next position */
         ref = LZ4_getPosition(ip, ctx, tableType, base);
+        if (dict==usingExtDict)
+        {
+            if (ref<(const BYTE*)source)
+            {
+                refDelta = dictDelta;
+                lowLimit = dictionary;
+            }
+            else
+            {
+                refDelta = 0;
+                lowLimit = (const BYTE*)source;
+            }
+        }
         LZ4_putPosition(ip, ctx, tableType, base);
-        if ((ref + MAX_DISTANCE >= ip) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; }
+        if ( (ref+MAX_DISTANCE>=ip) && (A32(ref+refDelta)==A32(ip)) ) { token=op++; *token=0; goto _next_match; }
 
         /* Prepare next loop */
-        anchor = ip++;
-        forwardH = LZ4_hashPosition(ip, tableType);
+        forwardH = LZ4_hashPosition(++ip, tableType);
     }
 
 _last_literals:
     /* Encode Last Literals */
     {
         int lastRun = (int)(iend - anchor);
-        if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0;   /* Check output limit */
+        if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0;   /* Check output limit */
         if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
         else *op++ = (BYTE)(lastRun<<ML_BITS);
         memcpy(op, anchor, iend - anchor);
@@ -557,9 +621,9 @@ _last_literals:
 int LZ4_compress(const char* source, char* dest, int inputSize)
 {
 #if (HEAPMODE)
-    void* ctx = ALLOCATOR(HASHNBCELLS4, 4);   /* Aligned on 4-bytes boundaries */
+    void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U32, 4);   /* Aligned on 4-bytes boundaries */
 #else
-    U32 ctx[1U<<(MEMORY_USAGE-2)] = {0};      /* Ensure data is aligned on 4-bytes boundaries */
+    U32 ctx[LZ4_STREAMSIZE_U32] = {0};      /* Ensure data is aligned on 4-bytes boundaries */
 #endif
     int result;
 
@@ -577,16 +641,16 @@ int LZ4_compress(const char* source, char* dest, int inputSize)
 int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
 {
 #if (HEAPMODE)
-    void* ctx = ALLOCATOR(HASHNBCELLS4, 4);   /* Aligned on 4-bytes boundaries */
+    void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U32, 4);   /* Aligned on 4-bytes boundaries */
 #else
-    U32 ctx[1U<<(MEMORY_USAGE-2)] = {0};      /* Ensure data is aligned on 4-bytes boundaries */
+    U32 ctx[LZ4_STREAMSIZE_U32] = {0};      /* Ensure data is aligned on 4-bytes boundaries */
 #endif
     int result;
 
     if (inputSize < (int)LZ4_64KLIMIT)
-        result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, byU16, noDict);
+        result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict);
     else
-        result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
+        result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
 
 #if (HEAPMODE)
     FREEMEM(ctx);
@@ -595,118 +659,169 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in
 }
 
 
-/*****************************
-   Using external allocation
-*****************************/
-
-int LZ4_sizeofState() { return 1 << MEMORY_USAGE; }
-
+/*****************************************
+   Experimental : Streaming functions
+*****************************************/
 
-int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize)
+void* LZ4_createStream()
 {
-    if (((size_t)(state)&3) != 0) return 0;   /* Error : state is not aligned on 4-bytes boundary */
-    MEM_INIT(state, 0, LZ4_sizeofState());
+    void* lz4s = ALLOCATOR(4, LZ4_STREAMSIZE_U32);
+    MEM_INIT(lz4s, 0, LZ4_STREAMSIZE);
+    return lz4s;
+}
 
-    if (inputSize < (int)LZ4_64KLIMIT)
-        return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict);
-    else
-        return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
+int LZ4_free (void* LZ4_stream)
+{
+    FREEMEM(LZ4_stream);
+    return (0);
 }
 
 
-int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
+int LZ4_loadDict (void* LZ4_dict, const char* dictionary, int dictSize)
 {
-    if (((size_t)(state)&3) != 0) return 0;   /* Error : state is not aligned on 4-bytes boundary */
-    MEM_INIT(state, 0, LZ4_sizeofState());
+    LZ4_dict_t_internal* dict = (LZ4_dict_t_internal*) LZ4_dict;
+    const BYTE* p = (const BYTE*)dictionary;
+    const BYTE* const dictEnd = p + dictSize;
+    const BYTE* base;
 
-    if (inputSize < (int)LZ4_64KLIMIT)
-        return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limited, byU16, noDict);
-    else
-        return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
-}
+    LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_dict_t_internal));        /* A compilation error here means LZ4_STREAMSIZE is not large enough */
+    if (dict->initCheck) MEM_INIT(dict, 0, sizeof(LZ4_dict_t_internal));
 
+    if (dictSize < MINMATCH)
+    {
+        dict->dictionary = NULL;
+        dict->dictSize = 0;
+        return 1;
+    }
 
-/****************************
-   Stream functions
-****************************/
+    if (p <= dictEnd - 64 KB) p = dictEnd - 64 KB;
+    base = p - dict->currentOffset;
+    dict->dictionary = p;
+    dict->dictSize = (U32)(dictEnd - p);
+    dict->currentOffset += dict->dictSize;
 
-int LZ4_sizeofStreamState()
-{
-    return sizeof(LZ4_Data_Structure);
-}
+    while (p <= dictEnd-MINMATCH)
+    {
+        LZ4_putPosition(p, dict, byU32, base);
+        p+=3;
+    }
 
-FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, const BYTE* base)
-{
-    MEM_INIT(lz4ds->hashTable, 0, sizeof(lz4ds->hashTable));
-    lz4ds->bufferStart = base;
-    lz4ds->base = base;
-    lz4ds->nextBlock = base;
+    return 1;
 }
 
-int LZ4_resetStreamState(void* state, const char* inputBuffer)
-{
-    if ((((size_t)state) & 3) != 0) return 1;   /* Error : pointer is not aligned on 4-bytes boundary */
-    LZ4_init((LZ4_Data_Structure*)state, (const BYTE*)inputBuffer);
-    return 0;
-}
 
-void* LZ4_create (const char* inputBuffer)
+void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict, const BYTE* src)
 {
-    void* lz4ds = ALLOCATOR(1, sizeof(LZ4_Data_Structure));
-    LZ4_init ((LZ4_Data_Structure*)lz4ds, (const BYTE*)inputBuffer);
-    return lz4ds;
+    if ((LZ4_dict->currentOffset > 0x80000000) ||
+        ((size_t)LZ4_dict->currentOffset > (size_t)src))   /* address space overflow */
+    {
+        /* rescale hash table */
+        U32 delta = LZ4_dict->currentOffset - 64 KB;
+        int i;
+        for (i=0; i<HASH_SIZE_U32; i++)
+        {
+            if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0;
+            else LZ4_dict->hashTable[i] -= delta;
+        }
+        LZ4_dict->currentOffset = 64 KB;
+        LZ4_dict->dictionary = LZ4_dict->dictionary + LZ4_dict->dictSize - 64 KB;
+        LZ4_dict->dictSize = 64 KB;
+    }
 }
 
 
-int LZ4_free (void* LZ4_Data)
+int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize)
 {
-    FREEMEM(LZ4_Data);
-    return (0);
-}
-
+    LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_stream;
+    const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
-char* LZ4_slideInputBuffer (void* LZ4_Data)
-{
-    LZ4_Data_Structure* lz4ds = (LZ4_Data_Structure*)LZ4_Data;
-    size_t delta = lz4ds->nextBlock - (lz4ds->bufferStart + 64 KB);
+    const BYTE* smallest = (const BYTE*) source;
+    if ((streamPtr->dictSize>0) && (smallest > dictEnd)) smallest = dictEnd;
+    LZ4_renormDictT(streamPtr, smallest);
 
-    if ( (lz4ds->base - delta > lz4ds->base)                          /* underflow control */
-       || ((size_t)(lz4ds->nextBlock - lz4ds->base) > 0xE0000000) )   /* close to 32-bits limit */
+    if (dictEnd == (const BYTE*)source)
     {
-        size_t deltaLimit = (lz4ds->nextBlock - 64 KB) - lz4ds->base;
-        int nH;
+        int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, withPrefix64k);
+        streamPtr->dictSize += (U32)inputSize;
+        streamPtr->currentOffset += (U32)inputSize;
+        return result;
+    }
 
-        for (nH=0; nH < HASHNBCELLS4; nH++)
-        {
-            if ((size_t)(lz4ds->hashTable[nH]) < deltaLimit) lz4ds->hashTable[nH] = 0;
-            else lz4ds->hashTable[nH] -= (U32)deltaLimit;
-        }
-        memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB);
-        lz4ds->base = lz4ds->bufferStart;
-        lz4ds->nextBlock = lz4ds->base + 64 KB;
+    {
+        int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, usingExtDict);
+        streamPtr->dictionary = (const BYTE*)source;
+        streamPtr->dictSize = (U32)inputSize;
+        streamPtr->currentOffset += (U32)inputSize;
+        return result;
     }
-    else
+}
+
+int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+    LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_stream;
+    const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
+
+    const BYTE* smallest = (const BYTE*) source;
+    if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd;
+    LZ4_renormDictT(streamPtr, smallest);
+
+    if (dictEnd == (const BYTE*)source)
     {
-        memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB);
-        lz4ds->nextBlock -= delta;
-        lz4ds->base -= delta;
+        int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k);
+        streamPtr->dictSize += (U32)inputSize;
+        streamPtr->currentOffset += (U32)inputSize;
+        return result;
     }
 
-    return (char*)(lz4ds->nextBlock);
+    {
+        int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict);
+        streamPtr->dictionary = (const BYTE*)source;
+        streamPtr->dictSize = (U32)inputSize;
+        streamPtr->currentOffset += (U32)inputSize;
+        return result;
+    }
 }
 
 
-int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize)
+// Hidden debug function, to force separate dictionary mode
+int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize)
 {
-    return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, 0, notLimited, byU32, withPrefix64k);
+    LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_dict;
+    int result;
+    const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
+
+    const BYTE* smallest = dictEnd;
+    if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
+    LZ4_renormDictT((LZ4_dict_t_internal*)LZ4_dict, smallest);
+
+    result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict);
+
+    streamPtr->dictionary = (const BYTE*)source;
+    streamPtr->dictSize = (U32)inputSize;
+    streamPtr->currentOffset += (U32)inputSize;
+
+    return result;
 }
 
-int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
+
+int LZ4_moveDict (void* LZ4_dict, char* safeBuffer, int dictSize)
 {
-    return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, maxOutputSize, limited, byU32, withPrefix64k);
+    LZ4_dict_t_internal* dict = (LZ4_dict_t_internal*) LZ4_dict;
+    const BYTE* previousDictEnd = dict->dictionary + dict->dictSize;
+
+    if ((U32)dictSize > 64 KB) dictSize = 64 KB;   /* useless to define a dictionary > 64 KB */
+    if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize;
+
+    memcpy(safeBuffer, previousDictEnd - dictSize, dictSize);
+
+    dict->dictionary = (const BYTE*)safeBuffer;
+    dict->dictSize = (U32)dictSize;
+
+    return 1;
 }
 
 
+
 /****************************
    Decompression functions
 ****************************/
@@ -716,7 +831,7 @@ int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, cha
  * Note that it is essential this generic function is really inlined,
  * in order to remove useless branches during compilation optimisation.
  */
-FORCE_INLINE int LZ4_decompress_generic(
+static int LZ4_decompress_generic(
                  const char* source,
                  char* dest,
                  int inputSize,
@@ -725,9 +840,9 @@ FORCE_INLINE int LZ4_decompress_generic(
                  int endOnInput,         /* endOnOutputSize, endOnInputSize */
                  int partialDecoding,    /* full, partial */
                  int targetOutputSize,   /* only used if partialDecoding==partial */
-                 int dict,               /* noDict, withPrefix64k, withExtDict */
-                 const char* dictStart,  /* only if dict==withExtDict */
-                 int dictSize            /* only if dict==withExtDict */
+                 int dict,               /* noDict, withPrefix64k, usingExtDict */
+                 const char* dictStart,  /* only if dict==usingExtDict */
+                 int dictSize            /* only if dict==usingExtDict */
                  )
 {
     /* Local Variables */
@@ -740,7 +855,7 @@ FORCE_INLINE int LZ4_decompress_generic(
     BYTE* cpy;
     BYTE* oexit = op + targetOutputSize;
 
-    const BYTE* const dictEnd = (dict==withExtDict) ? (const BYTE*)dictStart + dictSize : NULL;
+    const BYTE* const dictEnd = (dict==usingExtDict) ? (const BYTE*)dictStart + dictSize : NULL;
 
     const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0};   /* static reduces speed for LZ4_decompress_safe() on GCC64 */
     static const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3};
@@ -748,11 +863,10 @@ FORCE_INLINE int LZ4_decompress_generic(
 
     /* Special cases */
     (void)dictStart; (void)dictSize;
-    if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT;                        /* targetOutputSize too high => decode everything */
+    if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT;   /* targetOutputSize too high => decode everything */
     if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1;   /* Empty output buffer */
     if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
 
-
     /* Main Loop */
     while (1)
     {
@@ -764,11 +878,7 @@ FORCE_INLINE int LZ4_decompress_generic(
         if ((length=(token>>ML_BITS)) == RUN_MASK)
         {
             unsigned s=255;
-            while (((endOnInput)?ip<iend:1) && (s==255))
-            {
-                s = *ip++;
-                length += s;
-            }
+            while (((endOnInput)?ip<iend:1) && (s==255)) { s = *ip++; length += s; }
         }
 
         /* copy literals */
@@ -814,7 +924,7 @@ FORCE_INLINE int LZ4_decompress_generic(
         }
 
         /* check external dictionary */
-        if ((dict==withExtDict) && (unlikely(ref < (BYTE* const)dest)))
+        if ((dict==usingExtDict) && (ref < (BYTE* const)dest))
         {
             if (unlikely(op+length+MINMATCH > oend-LASTLITERALS)) goto _output_error;
 
@@ -830,7 +940,7 @@ FORCE_INLINE int LZ4_decompress_generic(
                 memcpy(op, dictEnd - copySize, copySize);
                 op += copySize;
                 copySize = length+MINMATCH - copySize;
-                if (copySize > (size_t)((char*)op-dest))
+                if (copySize > (size_t)((char*)op-dest))   /* overlap */
                 {
                     BYTE* const cpy = op + copySize;
                     const BYTE* ref = (BYTE*)dest;
@@ -862,7 +972,7 @@ FORCE_INLINE int LZ4_decompress_generic(
         if (unlikely(cpy>oend-COPYLENGTH-(STEPSIZE-4)))
         {
             if (cpy > oend-LASTLITERALS) goto _output_error;    /* Error : last 5 bytes must be literals */
-            LZ4_SECURECOPY(op, ref, (oend-COPYLENGTH));
+            if (op<oend-COPYLENGTH) LZ4_WILDCOPY(op, ref, (oend-COPYLENGTH));
             while(op<cpy) *op++=*ref++;
             op=cpy;
             continue;
@@ -895,7 +1005,7 @@ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compre
 
 int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
 {
-    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withExtDict, dictStart, dictSize);
+    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize);
 }
 
 int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxOutputSize)
@@ -919,7 +1029,7 @@ int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int origin
 
 int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
 {
-    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withExtDict, dictStart, dictSize);
+    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, dictStart, dictSize);
 }
 
 
@@ -927,8 +1037,71 @@ int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSi
    Obsolete Functions
 **************************************/
 /*
-These functions are deprecated and should no longer be used.
-They are provided here for compatibility with existing user programs.
+These function names are deprecated and should no longer be used.
+They are only provided here for compatibility with older user programs.
+- LZ4_uncompress is totally equivalent to LZ4_decompress_fast
+- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
 */
 int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
 int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+
+
+/* Obsolete Streaming functions */
+
+int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
+
+void LZ4_init(LZ4_dict_t_internal* lz4ds, const BYTE* base)
+{
+    MEM_INIT(lz4ds->hashTable, 0, LZ4_STREAMSIZE);
+    lz4ds->bufferStart = base;
+}
+
+int LZ4_resetStreamState(void* state, const char* inputBuffer)
+{
+    if ((((size_t)state) & 3) != 0) return 1;   /* Error : pointer is not aligned on 4-bytes boundary */
+    LZ4_init((LZ4_dict_t_internal*)state, (const BYTE*)inputBuffer);
+    return 0;
+}
+
+void* LZ4_create (const char* inputBuffer)
+{
+    void* lz4ds = ALLOCATOR(4, LZ4_STREAMSIZE_U32);
+    LZ4_init ((LZ4_dict_t_internal*)lz4ds, (const BYTE*)inputBuffer);
+    return lz4ds;
+}
+
+char* LZ4_slideInputBuffer (void* LZ4_Data)
+{
+    LZ4_dict_t_internal* lz4ds = (LZ4_dict_t_internal*)LZ4_Data;
+
+    LZ4_moveDict((LZ4_stream_t*)LZ4_Data, (char*)lz4ds->bufferStart, 64 KB);
+
+    return (char*)(lz4ds->bufferStart + 64 KB);
+}
+
+/*  User-allocated state */
+
+int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
+
+int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize)
+{
+    if (((size_t)(state)&3) != 0) return 0;   /* Error : state is not aligned on 4-bytes boundary */
+    MEM_INIT(state, 0, LZ4_sizeofState());
+
+    if (inputSize < (int)LZ4_64KLIMIT)
+        return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict);
+    else
+        return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
+}
+
+int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+    if (((size_t)(state)&3) != 0) return 0;   /* Error : state is not aligned on 4-bytes boundary */
+    MEM_INIT(state, 0, LZ4_sizeofState());
+
+    if (inputSize < (int)LZ4_64KLIMIT)
+        return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict);
+    else
+        return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
+}
+
diff --git a/lz4.h b/lz4.h
index 37bd282..f2975d4 100644
--- a/lz4.h
+++ b/lz4.h
@@ -28,8 +28,8 @@
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    You can contact the author at :
-   - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
    - LZ4 source repository : http://code.google.com/p/lz4/
+   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 #pragma once
 
@@ -47,11 +47,16 @@ extern "C" {
 
 
 /**************************************
-   Compiler Options
+   Tuning parameter
 **************************************/
-#if (defined(__GNUC__) && defined(__STRICT_ANSI__)) || (defined(_MSC_VER) && !defined(__cplusplus))   /* Visual Studio */
-#  define inline __inline           /* Visual C is not C99, but supports some kind of inline */
-#endif
+/*
+ * LZ4_MEMORY_USAGE :
+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+ * Increasing memory usage improves compression ratio
+ * Reduced memory usage can improve speed, due to cache effect
+ * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
+ */
+#define LZ4_MEMORY_USAGE 14
 
 
 /**************************************
@@ -72,10 +77,21 @@ LZ4_compress() :
              or 0 if the compression fails
 
 LZ4_decompress_safe() :
-    maxOutputSize : is the size of the destination buffer (which must be already allocated)
+    compressedSize : is obviously the source size
+    maxOutputSize : is the size of the destination buffer, which must be already allocated.
     return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
+             If the destination buffer is not large enough, decoding will stop and output an error code (<0).
              If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets
+             This function is protected against buffer overflow exploits :
+             it never writes outside of output buffer, and never reads outside of input buffer.
+             Therefore, it is protected against malicious data packets.
+*/
+
+
+/*
+Note :
+    Should you prefer to explicitly allocate compression-table memory using your own allocation method,
+    use the streaming functions provided below, simply reset the memory area between each call to LZ4_compress_continue()
 */
 
 
@@ -89,7 +105,6 @@ LZ4_decompress_safe() :
 LZ4_compressBound() :
     Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
     primarily useful for memory allocation of output buffer.
-    inline function is recommended for the general case,
     macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation).
 
     isize  : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE
@@ -142,84 +157,70 @@ LZ4_decompress_safe_partial() :
 int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxOutputSize);
 
 
-/*
-These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
-To know how much memory must be allocated for the compression tables, use :
-int LZ4_sizeofState();
-
-Note that tables must be aligned on 4-bytes boundaries, otherwise compression will fail (return code 0).
-
-The allocated memory can be provided to the compressions functions using 'void* state' parameter.
-LZ4_compress_withState() and LZ4_compress_limitedOutput_withState() are equivalent to previously described functions.
-They just use the externally allocated memory area instead of allocating their own one (on stack, or on heap).
-*/
-int LZ4_sizeofState(void);
-int LZ4_compress_withState               (void* state, const char* source, char* dest, int inputSize);
-int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-
-
 /**************************************
-   Streaming Functions
+   Experimental Streaming Functions
 **************************************/
-void* LZ4_create (const char* inputBuffer);
-int   LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize);
-int   LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
-char* LZ4_slideInputBuffer (void* LZ4_Data);
-int   LZ4_free (void* LZ4_Data);
 
+#define LZ4_STREAMSIZE_U32 ((1 << (LZ4_MEMORY_USAGE-2)) + 8)
+#define LZ4_STREAMSIZE     (LZ4_STREAMSIZE_U32 * sizeof(unsigned int))
 /*
-These functions allow the compression of chained blocks, where each block benefits from prior 64 KB within preceding blocks.
-In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function :
+ * LZ4_stream_t
+ * information structure to track an LZ4 stream.
+ * set it to zero, or use LZ4_loadDict() to init it before first use.
+ */
+typedef struct { unsigned int table[LZ4_STREAMSIZE_U32]; } LZ4_stream_t;
 
-void* LZ4_create (const char* inputBuffer);
-The result of the function is the (void*) pointer on the LZ4 Data Structure.
-This pointer will be needed in all other functions.
-If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
-The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
-The input buffer must be already allocated, and size at least 192KB.
-'inputBuffer' will also be the 'const char* source' of the first block.
-
-All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
-To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue().
-Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(),
-but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one.
-If next block does not begin immediately after the previous one, the compression will fail (return 0).
-
-When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
-char* LZ4_slideInputBuffer(void* LZ4_Data);
-must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
-Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
-==> The memory position where the next input data block must start is provided as the result of the function.
-
-Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual.
-
-When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure.
-*/
 
+/*
+ * If you prefer dynamic allocation methods,
+ * LZ4_createStream
+ * provides a pointer (void*) towards an initialized LZ4_stream_t structure.
+ * LZ4_free just frees it.
+ */
+void* LZ4_createStream();
+int   LZ4_free (void* LZ4_stream);
 
 /*
-The following functions achieve the same result as :
-void* LZ4_create (const char* inputBuffer);
+ * LZ4_loadDict
+ * Use this function to load a static dictionary into LZ4_stream.
+ * Loading a size of 0 is allowed and init the LZ4_stream_t structure.
+ * Return : 1 if OK, 0 if error
+ */
+int LZ4_loadDict (void* LZ4_stream, const char* dictionary, int dictSize);
 
-They are provided here to allow the user program to allocate memory using its own routines.
+/*
+ * LZ4_compress_continue
+ * Compress data block 'source', using blocks compressed before to improve compression ratio
+ * Previous data blocks are assumed to still be present at their previous location.
+ */
+int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize);
 
-To know how much space must be allocated, use LZ4_sizeofStreamState();
-Note also that space must be 4-bytes aligned.
+/*
+ * LZ4_compress_limitedOutput_continue
+ * Same as before, but also specify a maximum target compressed size (maxOutputSize)
+ * If it cannot be met, compression exits, and return a zero.
+ */
+int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize);
 
-Once space is allocated, you must initialize it using : LZ4_resetStreamState(void* state, const char* inputBuffer);
-void* state is a pointer to the space allocated.
-It must be aligned on 4-bytes boundaries, and be large enough.
-The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
-The input buffer must be already allocated, and size at least 192KB.
-'inputBuffer' will also be the 'const char* source' of the first block.
+/*
+ * LZ4_moveDict
+ * If previously compressed data block is not guaranteed to remain at its previous memory location
+ * save it into a safe place (char* safeBuffer)
+ * before calling again LZ4_compress_continue()
+ * Return : 1 if OK, 0 if error
+ * Note : any dictSize > 64 KB will be interpreted as 64KB.
+ */
+int LZ4_moveDict (void* LZ4_stream, char* safeBuffer, int dictSize);
 
-The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
-return value of LZ4_resetStreamState() must be 0 is OK.
-Any other value means there was an error (typically, pointer is not aligned on 4-bytes boundaries).
-*/
-int LZ4_sizeofStreamState(void);
-int LZ4_resetStreamState(void* state, const char* inputBuffer);
 
+/*
+*_usingDict() :
+    These decoding functions work the same as their "normal" versions,
+    but can also use up to 64KB of dictionary data (dictStart, dictSize)
+    to decode chained blocks.
+*/
+int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize);
+int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
 
 /*
 *_withPrefix64k() :
@@ -232,30 +233,31 @@ int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compr
 int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize);
 
 
-/**************************************
-   Experimental Functions
-**************************************/
-/*
-*_withDict() :
-    These decoding functions work the same as their "normal" versions,
-    but can also use up to 64KB of dictionary data
-    to decode chained blocks.
-*/
-int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize);
-int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
-
-
 
 /**************************************
    Obsolete Functions
 **************************************/
 /*
-These functions are deprecated and should no longer be used.
-They are provided here for compatibility with existing user programs.
+Obsolete decompression functions
+These function names are deprecated and should no longer be used.
+They are only provided here for compatibility with older user programs.
+- LZ4_uncompress is the same as LZ4_decompress_fast
+- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
 */
 int LZ4_uncompress (const char* source, char* dest, int outputSize);
 int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
+/* Obsolete external allocation functions */
+int LZ4_sizeofState(void);
+int LZ4_compress_withState               (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/* Obsolete streaming functions */
+void* LZ4_create (const char* inputBuffer);
+int   LZ4_sizeofStreamState(void);
+int   LZ4_resetStreamState(void* state, const char* inputBuffer);
+char* LZ4_slideInputBuffer (void* state);
+
 
 #if defined (__cplusplus)
 }
diff --git a/lz4hc.c b/lz4hc.c
index e84de2b..6086749 100644
--- a/lz4hc.c
+++ b/lz4hc.c
@@ -54,8 +54,9 @@
 **************************************/
 /* 32 or 64 bits ? */
 #if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
-  || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
-  || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
+  || defined(__powerpc64__) || defined(__powerpc64le__) \
+  || defined(__ppc64__) || defined(__ppc64le__) \
+  || defined(__PPC64__) || defined(__PPC64LE__) \
   || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) )   /* Detects 64 bits mode */
 #  define LZ4_ARCH64 1
 #else
@@ -66,6 +67,7 @@
  * Little Endian or Big Endian ?
  * Overwrite the #define below if you know your architecture endianess
  */
+#include <stdlib.h>   /* Apparently required to detect endianess */
 #if defined (__GLIBC__)
 #  include <endian.h>
 #  if (__BYTE_ORDER == __BIG_ENDIAN)
diff --git a/lz4hc.h b/lz4hc.h
index b810978..deb2394 100644
--- a/lz4hc.h
+++ b/lz4hc.h
@@ -104,6 +104,7 @@ They just use the externally allocated memory area instead of allocating their o
 /**************************************
    Streaming Functions
 **************************************/
+/* Note : these streaming functions still follows the older model */
 void* LZ4_createHC (const char* inputBuffer);
 int   LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
 int   LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
diff --git a/programs/Makefile b/programs/Makefile
index c0d6d15..811dda2 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -42,27 +42,32 @@ MANDIR=$(PREFIX)/share/man/man1
 LZ4DIR=..
 
 TEST_FILES = COPYING
-TEST_TARGETS=test-32 test-64
+TEST_TARGETS=test-64 test-32
+BENCH_NB=-i5
 
 # Minimize test target for Travis CI's Build Matrix
 ifeq ($(LZ4_TRAVIS_CI_ENV),-m32)
 TEST_TARGETS=test-32
+BENCH_NB=-i1
 else ifeq ($(LZ4_TRAVIS_CI_ENV),-m64)
 TEST_TARGETS=test-64
+BENCH_NB=-i1
 endif
 
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
 EXT =.exe
+VOID = nul
 else
 EXT =
+VOID = /dev/null
 endif
 
 
 default: lz4 lz4c
 
-all: lz4 lz4c lz4c32 fuzzer fuzzer32 fullbench fullbench32
+all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 datagen
 
 lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
 	$(CC)      $(FLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
@@ -73,22 +78,27 @@ lz4c  : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
 lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
 	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
 
+fullbench  : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+
+fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c
+	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
+
 fuzzer  : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c 
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
 fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c
 	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
 
-fullbench  : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c
+datagen : datagen.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c
-	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
 
 clean:
 	@rm -f core *.o \
         lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
-        fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT) 
+        fullbench$(EXT) fullbench32$(EXT) \
+        fuzzer$(EXT) fuzzer32$(EXT) datagen$(EXT)
 	@echo Cleaning completed
 
 
@@ -117,21 +127,29 @@ uninstall:
 
 test: $(TEST_TARGETS)
 
-test-32: test-lz4 test-lz4c32 test-fullbench32 test-fuzzer32
+test-64: test-lz4 test-lz4c test-fullbench test-fuzzer test-mem
 
-test-64: test-lz4 test-lz4c test-fullbench test-fuzzer
+test-32: test-lz4 test-lz4c32 test-fullbench32 test-fuzzer32 test-mem32
 
-test-lz4:
+test-lz4: lz4 datagen
+	./datagen | ./lz4 | ./lz4 -vdq > $(VOID)
+	./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID)
+	./datagen -g6GB   | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID)
 
-test-lz4c:
+test-lz4c: lz4c datagen
 
-test-lz4c32:
+test-lz4c32: lz4c32 lz4 datagen
+	./datagen | ./lz4c32 | ./lz4c32 -vdq > $(VOID)
+	./datagen | ./lz4c32 | ./lz4    -vdq > $(VOID)
+	./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -vdq > $(VOID)
+	./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4    -vdq > $(VOID)
+	./datagen -g6GB   | ./lz4c32 -vqB5D | ./lz4c32 -vdq > $(VOID)
 
 test-fullbench: fullbench
-	./fullbench --no-prompt $(TEST_FILES)
+	./fullbench --no-prompt $(BENCH_NB) $(TEST_FILES)
 
 test-fullbench32: fullbench32
-	./fullbench32 --no-prompt $(TEST_FILES)
+	./fullbench32 --no-prompt $(BENCH_NB) $(TEST_FILES)
 
 test-fuzzer: fuzzer
 	./fuzzer --no-prompt
@@ -139,5 +157,14 @@ test-fuzzer: fuzzer
 test-fuzzer32: fuzzer32
 	./fuzzer32 --no-prompt
 
+test-mem: lz4 datagen
+	./datagen -g256MB > tmp
+	valgrind ./lz4 -B4D -f tmp /dev/null
+	./datagen -g16MB > tmp
+	valgrind ./lz4 -9 -B5D -f tmp /dev/null
+	rm tmp
+
+test-mem32: lz4c32 datagen
+# unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system...
 
 endif
diff --git a/programs/datagen.c b/programs/datagen.c
new file mode 100644
index 0000000..0109d54
--- /dev/null
+++ b/programs/datagen.c
@@ -0,0 +1,283 @@
+/*
+    datagen.c - compressible data generator test tool
+    Copyright (C) Yann Collet 2012-2014
+    GPL v2 License
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    You can contact the author at :
+    - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+    - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+
+/**************************************
+ Remove Visual warning messages
+**************************************/
+#define _CRT_SECURE_NO_WARNINGS   // fgets
+
+
+/**************************************
+ Includes
+**************************************/
+//#include <stdlib.h>
+#include <stdio.h>      // fgets, sscanf
+#include <string.h>     // strcmp
+
+
+/**************************************
+   Basic Types
+**************************************/
+#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)   /* C99 */
+# include <stdint.h>
+  typedef  uint8_t BYTE;
+  typedef uint16_t U16;
+  typedef uint32_t U32;
+  typedef  int32_t S32;
+  typedef uint64_t U64;
+#else
+  typedef unsigned char       BYTE;
+  typedef unsigned short      U16;
+  typedef unsigned int        U32;
+  typedef   signed int        S32;
+  typedef unsigned long long  U64;
+#endif
+
+
+/**************************************
+ Constants
+**************************************/
+#ifndef LZ4_VERSION
+#  define LZ4_VERSION "rc118"
+#endif
+
+#define KB *(1U<<10)
+#define MB *(1U<<20)
+#define GB *(1U<<30)
+
+#define CDG_SIZE_DEFAULT (64 KB)
+#define CDG_SEED_DEFAULT 0
+#define CDG_COMPRESSIBILITY_DEFAULT 50
+#define PRIME1   2654435761U
+#define PRIME2   2246822519U
+
+
+/**************************************
+  Macros
+**************************************/
+#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+
+
+/**************************************
+  Local Parameters
+**************************************/
+static int no_prompt = 0;
+static char* programName;
+static int displayLevel = 2;
+
+
+/*********************************************************
+  Fuzzer functions
+*********************************************************/
+
+#define CDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+static unsigned int CDG_rand(U32* src)
+{
+    U32 rand32 = *src;
+    rand32 *= PRIME1;
+    rand32 += PRIME2;
+    rand32  = CDG_rotl32(rand32, 13);
+    *src = rand32;
+    return rand32;
+}
+
+
+#define CDG_RAND15BITS  ((CDG_rand(seed) >> 3) & 32767)
+#define CDG_RANDLENGTH  ( ((CDG_rand(seed) >> 7) & 3) ? (CDG_rand(seed) % 14) : (CDG_rand(seed) & 511) + 15)
+#define CDG_RANDCHAR    (((CDG_rand(seed) >> 9) & 63) + '0')
+static void CDG_generate(U64 size, U32* seed, double proba)
+{
+    BYTE fullbuff[32 KB + 128 KB + 1];
+    BYTE* buff = fullbuff + 32 KB;
+    U64 total=0;
+    U32 P32 = (U32)(32768 * proba);
+    U32 pos=0;
+    U32 genBlockSize = 128 KB;
+
+    // Build initial prefix
+    while (pos<32 KB)
+    {
+        // Select : Literal (char) or Match (within 32K)
+        if (CDG_RAND15BITS < P32)
+        {
+            // Copy (within 64K)
+            U32 d;
+            int ref;
+            int length = CDG_RANDLENGTH + 4;
+            U32 offset = CDG_RAND15BITS + 1;
+            if (offset > pos) offset = pos;
+            ref = pos - offset;
+            d = pos + length;
+            while (pos < d) fullbuff[pos++] = fullbuff[ref++];
+        }
+        else
+        {
+            // Literal (noise)
+            U32 d;
+            int length = CDG_RANDLENGTH;
+            d = pos + length;
+            while (pos < d) fullbuff[pos++] = CDG_RANDCHAR;
+        }
+    }
+
+    // Generate compressible data
+    pos = 0;
+    while (total < size)
+    {
+        if (size-total < 128 KB) genBlockSize = (U32)(size-total);
+        total += genBlockSize;
+        buff[genBlockSize] = 0;
+        pos = 0;
+        while (pos<genBlockSize)
+        {
+            // Select : Literal (char) or Match (within 32K)
+            if (CDG_RAND15BITS < P32)
+            {
+                // Copy (within 64K)
+                int ref;
+                U32 d;
+                int length = CDG_RANDLENGTH + 4;
+                U32 offset = CDG_RAND15BITS + 1;
+                if (pos + length > genBlockSize ) length = genBlockSize - pos;
+                ref = pos - offset;
+                d = pos + length;
+                while (pos < d) buff[pos++] = buff[ref++];
+            }
+            else
+            {
+                // Literal (noise)
+                U32 d;
+                int length = CDG_RANDLENGTH;
+                if (pos + length > genBlockSize) length = genBlockSize - pos;
+                d = pos + length;
+                while (pos < d) buff[pos++] = CDG_RANDCHAR;
+            }
+        }
+        pos=0;
+        for (;pos+512<=genBlockSize;pos+=512) printf("%512.512s", buff+pos);
+        for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
+        // Regenerate prefix
+        memcpy(fullbuff, buff + 96 KB, 32 KB);
+    }
+}
+
+
+int CDG_usage()
+{
+    DISPLAY( "Compressible data generator\n");
+    DISPLAY( "Usage :\n");
+    DISPLAY( "      %s [size] [args]\n", programName);
+    DISPLAY( "\n");
+    DISPLAY( "Arguments :\n");
+    DISPLAY( " -g#    : generate # data (default:%i)\n", CDG_SIZE_DEFAULT);
+    DISPLAY( " -s#    : Select seed (default:%i)\n", CDG_SEED_DEFAULT);
+    DISPLAY( " -p#    : Select compressibility in %% (default:%i%%)\n", CDG_COMPRESSIBILITY_DEFAULT);
+    DISPLAY( " -h     : display help and exit\n");
+    return 0;
+}
+
+
+int main(int argc, char** argv)
+{
+    int argNb;
+    int proba = CDG_COMPRESSIBILITY_DEFAULT;
+    U64 size = CDG_SIZE_DEFAULT;
+    U32 seed = CDG_SEED_DEFAULT;
+
+    // Check command line
+    programName = argv[0];
+    for(argNb=1; argNb<argc; argNb++)
+    {
+        char* argument = argv[argNb];
+
+        if(!argument) continue;   // Protection if argument empty
+
+        // Decode command (note : aggregated commands are allowed)
+        if (argument[0]=='-')
+        {
+            if (!strcmp(argument, "--no-prompt")) { no_prompt=1; continue; }
+
+            while (argument[1]!=0)
+            {
+                argument++;
+                switch(*argument)
+                {
+                case 'h':
+                    return CDG_usage();
+                case 'g':
+                    argument++;
+                    size=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        size *= 10;
+                        size += *argument - '0';
+                        argument++;
+                    }
+                    if (*argument=='K') { size <<= 10; argument++; }
+                    if (*argument=='M') { size <<= 20; argument++; }
+                    if (*argument=='G') { size <<= 30; argument++; }
+                    if (*argument=='B') { argument++; }
+                    break;
+                case 's':
+                    argument++;
+                    seed=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        seed *= 10;
+                        seed += *argument - '0';
+                        argument++;
+                    }
+                    break;
+                case 'p':
+                    argument++;
+                    proba=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        proba *= 10;
+                        proba += *argument - '0';
+                        argument++;
+                    }
+                    if (proba<0) proba=0;
+                    if (proba>100) proba=100;
+                    break;
+                case 'v':
+                    displayLevel = 4;
+                    break;
+                default: ;
+                }
+            }
+
+        }
+    }
+
+    // Get Seed
+    DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION);
+    DISPLAYLEVEL(3, "Seed = %u \n", seed);
+    if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
+
+    CDG_generate(size, &seed, ((double)proba) / 100);
+
+    return 0;
+}
diff --git a/programs/fullbench.c b/programs/fullbench.c
index 1200ca0..154de78 100644..100755
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -254,88 +254,104 @@ static U64 BMK_GetFileSize(char* infilename)
   Benchmark function
 *********************************************************/
 
-static inline int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
+static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
 {
     return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
 }
 
 static void* stateLZ4;
-static inline int local_LZ4_compress_withState(const char* in, char* out, int inSize)
+static int local_LZ4_compress_withState(const char* in, char* out, int inSize)
 {
     return LZ4_compress_withState(stateLZ4, in, out, inSize);
 }
 
-static inline int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
+static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
 {
     return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize));
 }
 
 static void* ctx;
-static inline int local_LZ4_compress_continue(const char* in, char* out, int inSize)
+static int local_LZ4_compress_continue(const char* in, char* out, int inSize)
 {
     return LZ4_compress_continue(ctx, in, out, inSize);
 }
 
-static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
+static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
 {
     return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
 }
 
+
+LZ4_stream_t LZ4_dict;
+static void* local_LZ4_resetDictT(const char* fake)
+{
+    (void)fake;
+    memset(&LZ4_dict, 0, sizeof(LZ4_stream_t));
+    return NULL;
+}
+
+int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize);
+static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
+{
+    return LZ4_compress_forceExtDict(&LZ4_dict, in, out, inSize);
+}
+
+
 static void* stateLZ4HC;
-static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
+static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
 {
     return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize);
 }
 
-static inline int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
+static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
 {
     return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
 }
 
-static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
+static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
 {
     return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
 }
 
-static inline int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
+static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
 {
     return LZ4_compressHC_continue(ctx, in, out, inSize);
 }
 
-static inline int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
+static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
 {
     return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
 }
 
-static inline int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
+static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
     LZ4_decompress_fast(in, out, outSize);
     return outSize;
 }
 
-static inline int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
+static int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
     LZ4_decompress_fast_withPrefix64k(in, out, outSize);
     return outSize;
 }
 
-static inline int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize)
+static int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
     LZ4_decompress_fast_usingDict(in, out, outSize, in - 65536, 65536);
     return outSize;
 }
 
-static inline int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
+static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
     LZ4_decompress_safe_usingDict(in, out, inSize, outSize, in - 65536, 65536);
     return outSize;
 }
 
-static inline int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
+static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
 {
     return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
 }
@@ -344,23 +360,16 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
 {
   int fileIdx=0;
   char* orig_buff;
-# define NB_COMPRESSION_ALGORITHMS 12
+# define NB_COMPRESSION_ALGORITHMS 13
 # define MINCOMPRESSIONCHAR '0'
-# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS)
-  static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput",
-                                      "LZ4_compress_withState", "LZ4_compress_limitedOutput_withState",
-                                      "LZ4_compress_continue", "LZ4_compress_limitedOutput_continue",
-                                      "LZ4_compressHC", "LZ4_compressHC_limitedOutput",
-                                      "LZ4_compressHC_withStateHC", "LZ4_compressHC_limitedOutput_withStateHC",
-                                      "LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" };
-  double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0};
-  double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0};
+  double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0};
+  double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0};
 # define NB_DECOMPRESSION_ALGORITHMS 7
 # define MINDECOMPRESSIONCHAR '0'
 # define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS)
   static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_fast_usingDict",
                                         "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_usingDict", "LZ4_decompress_safe_partial" };
-  double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0};
+  double totalDTime[NB_DECOMPRESSION_ALGORITHMS+1] = {0};
 
   U64 totals = 0;
 
@@ -465,29 +474,30 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
         DISPLAY(" %s : \n", inFileName);
 
         // Compression Algorithms
-        for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
+        for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
         {
-            char* cName = compressionNames[cAlgNb];
+            char* compressorName;
             int (*compressionFunction)(const char*, char*, int);
             void* (*initFunction)(const char*) = NULL;
             double bestTime = 100000000.;
 
-            if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb+1)) continue;
+            if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue;
 
             switch(cAlgNb)
             {
-            case 0 : compressionFunction = LZ4_compress; break;
-            case 1 : compressionFunction = local_LZ4_compress_limitedOutput; break;
-            case 2 : compressionFunction = local_LZ4_compress_withState; break;
-            case 3 : compressionFunction = local_LZ4_compress_limitedOutput_withState; break;
-            case 4 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break;
-            case 5 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break;
-            case 6 : compressionFunction = LZ4_compressHC; break;
-            case 7 : compressionFunction = local_LZ4_compressHC_limitedOutput; break;
-            case 8 : compressionFunction = local_LZ4_compressHC_withStateHC; break;
-            case 9 : compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; break;
-            case 10: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break;
-            case 11: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break;
+            case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break;
+            case 2 : compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
+            case 3 : compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
+            case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
+            case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break;
+            case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break;
+            case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break;
+            case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
+            case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
+            case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
+            case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break;
+            case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
+            case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
             default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
             }
 
@@ -496,7 +506,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
                 double averageTime;
                 int milliTime;
 
-                PROGRESS("%1i-%-21.21s : %9i ->\r", loopNb, cName, (int)benchedSize);
+                PROGRESS("%1i-%-25.25s : %9i ->\r", loopNb, compressorName, (int)benchedSize);
                 { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; }     // warmimg up memory
 
                 nb_loops = 0;
@@ -509,7 +519,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
                     for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
                     {
                         chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
-                        if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", cName), exit(1);
+                        if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
                     }
                     if (initFunction!=NULL) free(ctx);
                     nb_loops++;
@@ -520,13 +530,13 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
                 if (averageTime < bestTime) bestTime = averageTime;
                 cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
                 ratio = (double)cSize/(double)benchedSize*100.;
-                PROGRESS("%1i-%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
+                PROGRESS("%1i-%-25.25s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
             }
 
             if (ratio<100.)
-                DISPLAY("%-23.23s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
+                DISPLAY("%-27.27s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
             else
-                DISPLAY("%-23.23s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
+                DISPLAY("%-27.27s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
 
             totalCTime[cAlgNb] += bestTime;
             totalCSize[cAlgNb] += cSize;
@@ -536,7 +546,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
         for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
         {
             chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
-            if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressionNames[0]), exit(1);
+            if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
         }
         { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; }     // zeroing source area, for CRC checking
 
@@ -558,7 +568,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
             case 4: decompressionFunction = LZ4_decompress_safe_withPrefix64k; break;
             case 5: decompressionFunction = local_LZ4_decompress_safe_usingDict; break;
             case 6: decompressionFunction = local_LZ4_decompress_safe_partial; break;
-            default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
+            default : DISPLAY("ERROR ! Bad decompression algorithm Id !! \n"); free(chunkP); return 1;
             }
 
             for (loopNb = 1; loopNb <= nbIterations; loopNb++)
@@ -607,25 +617,6 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
       free(chunkP);
   }
 
-  if (nbFiles > 1)
-  {
-      int AlgNb;
-
-      DISPLAY(" ** TOTAL ** : \n");
-      for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++)
-      {
-          char* cName = compressionNames[AlgNb];
-          if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue;
-          DISPLAY("%-23.23s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.);
-      }
-      for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++)
-      {
-          char* dName = decompressionNames[AlgNb];
-          if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue;
-          DISPLAY("%-31.31s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.);
-      }
-  }
-
   if (BMK_pause) { printf("press enter...\n"); getchar(); }
 
   return 0;
@@ -669,7 +660,7 @@ int main(int argc, char** argv)
     char* input_filename=0;
 
     // Welcome message
-    DISPLAY( WELCOME_MESSAGE);
+    DISPLAY(WELCOME_MESSAGE);
 
     if (argc<2) { badusage(exename); return 1; }
 
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index 1ea14f6..d209fd3 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -83,11 +83,12 @@
 #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 
 
-//**************************************
-// Local Parameters
-//**************************************
+/*****************************************
+  Local Parameters
+*****************************************/
 static int no_prompt = 0;
 static char* programName;
+static int displayLevel = 2;
 
 
 /*********************************************************
@@ -187,7 +188,7 @@ int FUZ_SecurityTest()
 
 #define FUZ_MAX(a,b) (a>b?a:b)
 
-int FUZ_test(U32 seed, int nbTests, double compressibility) {
+int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
         unsigned long long bytes = 0;
         unsigned long long cbytes = 0;
         unsigned long long hcbytes = 0;
@@ -197,14 +198,16 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
         char* decodedBuffer;
 #       define FUZ_max   LZ4_COMPRESSBOUND(LEN)
         unsigned int randState=seed;
-        int ret, attemptNb;
+        int ret, cycleNb;
 #       define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
-                                        printf(" (seed %u, cycle %i) \n", seed, attemptNb); goto _output_error; }
-#       define FUZ_DISPLAYTEST          testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb);
+                                        printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
+#       define FUZ_DISPLAYTEST          { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); }
         void* stateLZ4   = malloc(LZ4_sizeofState());
         void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
         void* LZ4continue;
+        LZ4_stream_t LZ4dict;
         U32 crcOrig, crcCheck;
+        int displayRefresh;
 
 
         // Create compressible test buffer
@@ -213,8 +216,25 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
         compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
         decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
 
+        // display refresh rate
+        switch(displayLevel)
+        {
+        case 0: displayRefresh = nbCycles+1; break;
+        case 1: displayRefresh=FUZ_MAX(1, nbCycles / 100); break;
+        case 2: displayRefresh=99; break;
+        default : displayRefresh=1;
+        }
+
+        // move to startCycle
+        for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
+        {
+            FUZ_rand(&randState);
+            FUZ_rand(&randState);
+            FUZ_rand(&randState);
+        }
+
         // Test loop
-        for (attemptNb = 0; attemptNb < nbTests; attemptNb++)
+        for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
         {
             int testNb = 0;
             char* dict;
@@ -222,14 +242,10 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
             int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize;
             int blockContinueCompressedSize;
 
-            // note : promptThrottle is throtting stdout to prevent
-            //        Travis-CI's output limit (10MB) and false hangup detection.
-            const int step = FUZ_MAX(1, nbTests / 100);
-            const int promptThrottle = ((attemptNb % step) == 0);
-            if (!no_prompt || attemptNb == 0 || promptThrottle)
+            if ((cycleNb%displayRefresh) == 0)
             {
-                printf("\r%7i /%7i   - ", attemptNb, nbTests);
-                if (no_prompt) fflush(stdout);
+                printf("\r%7i /%7i   - ", cycleNb, nbCycles);
+                fflush(stdout);
             }
 
             // Select block to test
@@ -305,7 +321,7 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
             ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
             FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space");
             FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
-            //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than target size");   // well, is that an issue ?
+            //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size");   // well, is that an issue ?
             FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
             crcCheck = XXH32(decodedBuffer, blockSize, 0);
             FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
@@ -401,7 +417,14 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
             ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
             crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
-            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data");
+            if (crcCheck!=crcOrig)
+            {
+                int i=0;
+                while (block[i]==decodedBuffer[i]) i++;
+                printf("Wrong Byte at position %i/%i\n", i, blockSize);
+
+            }
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
 
             FUZ_DISPLAYTEST;
             ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize);
@@ -409,6 +432,27 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
             crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
             FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
 
+            // Compress using dictionary
+            FUZ_DISPLAYTEST;
+            dict -= 9;
+            if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
+            memset(&LZ4dict, 0, sizeof(LZ4_stream_t));
+            LZ4_loadDict(&LZ4dict, dict, dictSize);
+            blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
+            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_usingDict failed");
+
+            FUZ_DISPLAYTEST;
+            memset(&LZ4dict, 0, sizeof(LZ4_stream_t));
+            LZ4_loadDict(&LZ4dict, dict, dictSize);
+            ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+            FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_usingDict should fail : one missing byte for output buffer");
+
+            FUZ_DISPLAYTEST;
+            memset(&LZ4dict, 0, sizeof(LZ4_stream_t));
+            LZ4_loadDict(&LZ4dict, dict, dictSize);
+            ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_usingDict should work : enough size available within output buffer");
+
             // Decompress with dictionary as external
             FUZ_DISPLAYTEST;
             decodedBuffer[blockSize] = 0;
@@ -416,7 +460,14 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
             FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
             crcCheck = XXH32(decodedBuffer, blockSize, 0);
-            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data");
+            if (crcCheck!=crcOrig)
+            {
+                int i=0;
+                while (block[i]==decodedBuffer[i]) i++;
+                printf("Wrong Byte at position %i/%i\n", i, blockSize);
+
+            }
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
 
             FUZ_DISPLAYTEST;
             decodedBuffer[blockSize] = 0;
@@ -455,7 +506,7 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
             ccbytes += blockContinueCompressedSize;
         }
 
-        printf("\r%7i /%7i   - ", attemptNb, nbTests);
+        printf("\r%7i /%7i   - ", cycleNb, nbCycles);
         printf("all tests completed successfully \n");
         printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
         printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
@@ -489,7 +540,9 @@ int FUZ_usage()
     DISPLAY( "Arguments :\n");
     DISPLAY( " -i#    : Nb of tests (default:%i) \n", NB_ATTEMPTS);
     DISPLAY( " -s#    : Select seed (default:prompt user)\n");
+    DISPLAY( " -t#    : Select starting test number (default:0)\n");
     DISPLAY( " -p#    : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
+    DISPLAY( " -v     : verbose\n");
     DISPLAY( " -h     : display help and exit\n");
     return 0;
 }
@@ -502,6 +555,7 @@ int main(int argc, char** argv) {
     int seedset=0;
     int argNb;
     int nbTests = NB_ATTEMPTS;
+    int testNb = 0;
     int proba = FUZ_COMPRESSIBILITY_DEFAULT;
 
     // Check command line
@@ -515,7 +569,7 @@ int main(int argc, char** argv) {
         // Decode command (note : aggregated commands are allowed)
         if (argument[0]=='-')
         {
-            if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; continue; }
+            if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; displayLevel=1; continue; }
 
             while (argument[1]!=0)
             {
@@ -524,6 +578,10 @@ int main(int argc, char** argv) {
                 {
                 case 'h':
                     return FUZ_usage();
+                case 'v':
+                    argument++;
+                    displayLevel=4;
+                    break;
                 case 'i':
                     argument++;
                     nbTests=0;
@@ -544,6 +602,16 @@ int main(int argc, char** argv) {
                         argument++;
                     }
                     break;
+                case 't':
+                    argument++;
+                    testNb=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        testNb *= 10;
+                        testNb += *argument - '0';
+                        argument++;
+                    }
+                    break;
                 case 'p':
                     argument++;
                     proba=0;
@@ -583,5 +651,5 @@ int main(int argc, char** argv) {
 
     if (nbTests<=0) nbTests=1;
 
-    return FUZ_test(seed, nbTests, ((double)proba) / 100);
+    return FUZ_test(seed, nbTests, testNb, ((double)proba) / 100);
 }
diff --git a/programs/lz4.1 b/programs/lz4.1
index 298cbf6..6ae8d3c 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -64,6 +64,7 @@ following options
 .TP
 .B \-B#
  block size [4-7](default : 7)
+ B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB
 .TP
 .B \-BD
  block dependency (improve compression ratio)
@@ -84,4 +85,4 @@ following options
 Report bugs at:- https://code.google.com/p/lz4/
 
 .SH AUTHOR
-Yann Collet
\ No newline at end of file
+Yann Collet
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 934c2bb..fd2721d 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -109,7 +109,7 @@
 //****************************
 #define COMPRESSOR_NAME "LZ4 Compression CLI"
 #ifndef LZ4_VERSION
-#  define LZ4_VERSION "v1.1.5"
+#  define LZ4_VERSION "v1.1.8"
 #endif
 #define AUTHOR "Yann Collet"
 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__
@@ -344,6 +344,19 @@ int main(int argc, char** argv)
                 if (*argument=='s') { displayLevel=1; continue; }                                          // -s (silent mode)
 #endif // DISABLE_LZ4C_LEGACY_OPTIONS
 
+                if ((*argument>='0') && (*argument<='9'))
+                {
+                    cLevel = 0;
+                    while ((*argument >= '0') && (*argument <= '9'))
+                    {
+                        cLevel *= 10;
+                        cLevel += *argument - '0';
+                        argument++;
+                    }
+                    argument--;
+                    continue;
+                }
+
                 switch(argument[0])
                 {
                     // Display help
@@ -354,20 +367,6 @@ int main(int argc, char** argv)
                     // Compression (default)
                 case 'z': forceCompress = 1; break;
 
-                    // Compression level
-                case '0':
-                case '1':
-                case '2':
-                case '3':
-                case '4':
-                case '5':
-                case '6':
-                case '7':
-                case '8':
-                case '9':
-                case 'A':   /* non documented (hidden) */
-                    cLevel=*argument -'0'; break;
-
                     // Use Legacy format (for Linux kernel compression)
                 case 'l': legacy_format=1; break;
 
@@ -405,9 +404,8 @@ int main(int argc, char** argv)
                         case '7':
                         {
                             int B = argument[1] - '0';
-                            int S = 1 << (8 + 2*B);
-                            BMK_SetBlocksize(S);
                             blockSize = LZ4IO_setBlockSizeID(B);
+                            BMK_SetBlocksize(blockSize);
                             argument++;
                             break;
                         }
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 05955cf..650681b 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -365,17 +365,37 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i
 }
 
 
+static void* LZ4IO_LZ4_createStream (const char* inputBuffer)
+{
+    (void)inputBuffer;
+    return LZ4_createStream();
+}
+
+static int LZ4IO_LZ4_compress_limitedOutput_continue (void* ctx, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
+{
+    (void)compressionLevel;
+    return LZ4_compress_limitedOutput_continue(ctx, source, dest, inputSize, maxOutputSize);
+}
+
+static int LZ4IO_LZ4_slideInputBufferHC (void* ctx, char* buffer, int size)
+{
+    (void)size; (void)buffer;
+    LZ4_slideInputBufferHC (ctx);
+    return 1;
+}
+
+
 static int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel)
 {
     void* (*initFunction)       (const char*);
-    int   (*compressionFunction)(void*, const char*, char*, int, int);
-    char* (*translateFunction)  (void*);
+    int   (*compressionFunction)(void*, const char*, char*, int, int, int);
+    int   (*nextBlockFunction)  (void*, char*, int);
     int   (*freeFunction)       (void*);
     void* ctx;
     unsigned long long filesize = 0;
     unsigned long long compressedfilesize = 0;
     unsigned int checkbits;
-    char* in_buff, *in_start, *in_end;
+    char* in_buff, *in_blockStart;
     char* out_buff;
     FILE* finput;
     FILE* foutput;
@@ -384,34 +404,35 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
     size_t sizeCheck, header_size;
     void* streamChecksumState=NULL;
 
-
     // Init
     start = clock();
     if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;
-    if (compressionlevel>=3)
+
+    if (compressionlevel<3)
     {
-        initFunction = LZ4_createHC;
-        compressionFunction = LZ4_compressHC_limitedOutput_continue;
-        translateFunction = LZ4_slideInputBufferHC;
-        freeFunction = LZ4_freeHC;
+        initFunction = LZ4IO_LZ4_createStream;
+        compressionFunction = LZ4IO_LZ4_compress_limitedOutput_continue;
+        nextBlockFunction = LZ4_moveDict;
+        freeFunction = LZ4_free;
     }
     else
     {
-        initFunction = LZ4_create;
-        compressionFunction = LZ4_compress_limitedOutput_continue;
-        translateFunction = LZ4_slideInputBuffer;
+        initFunction = LZ4_createHC;
+        compressionFunction = LZ4_compressHC2_limitedOutput_continue;
+        nextBlockFunction = LZ4IO_LZ4_slideInputBufferHC;
         freeFunction = LZ4_free;
     }
+
     get_fileHandle(input_filename, output_filename, &finput, &foutput);
     blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
 
     // Allocate Memory
-    inputBufferSize = blockSize + 64 KB;
-    if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE;
+    inputBufferSize = 64 KB + blockSize;
     in_buff  = (char*)malloc(inputBufferSize);
     out_buff = (char*)malloc(blockSize+CACHELINE);
     if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");
-    in_start = in_buff; in_end = in_buff + inputBufferSize;
+    in_blockStart = in_buff + 64 KB;
+    if (compressionlevel>=3) in_blockStart = in_buff;
     if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
     ctx = initFunction(in_buff);
 
@@ -435,19 +456,19 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
     {
         unsigned int outSize;
         unsigned int inSize;
+
         // Read Block
-        if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx);
-        inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput);
+        inSize = (unsigned int) fread(in_blockStart, (size_t)1, (size_t)blockSize, finput);
         if( inSize==0 ) break;   // No more input : end of compression
         filesize += inSize;
         DISPLAYLEVEL(3, "\rRead : %i MB   ", (int)(filesize>>20));
-        if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize);
+        if (streamChecksum) XXH32_update(streamChecksumState, in_blockStart, inSize);
 
         // Compress Block
-        outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1);
+        outSize = compressionFunction(ctx, in_blockStart, out_buff+4, inSize, inSize-1, compressionlevel);
         if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;
         if (blockChecksum) compressedfilesize+=4;
-        DISPLAYLEVEL(3, "\rRead : %i MB  ==> %.2f%%   ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
+        DISPLAYLEVEL(3, "==> %.2f%%   ", (double)compressedfilesize/filesize*100);
 
         // Write Block
         if (outSize > 0)
@@ -462,24 +483,28 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
             sizeToWrite = 4 + outSize + (4*blockChecksum);
             sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
             if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
-
         }
         else   // Copy Original
         {
             * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000);   // Add Uncompressed flag
             sizeCheck = fwrite(out_buff, 1, 4, foutput);
             if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
-            sizeCheck = fwrite(in_start, 1, inSize, foutput);
+            sizeCheck = fwrite(in_blockStart, 1, inSize, foutput);
             if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");
             if (blockChecksum)
             {
-                unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED);
+                unsigned int checksum = XXH32(in_blockStart, inSize, LZ4S_CHECKSUM_SEED);
                 * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
                 sizeCheck = fwrite(out_buff, 1, 4, foutput);
                 if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
             }
         }
-        in_start += inSize;
+        {
+            size_t sizeToMove = 64 KB;
+            if (inSize < 64 KB) sizeToMove = inSize;
+            nextBlockFunction(ctx, in_blockStart - sizeToMove, sizeToMove);
+            if (compressionlevel>=3) in_blockStart = in_buff + 64 KB;
+        }
     }
 
     // End of Stream mark
@@ -542,7 +567,7 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp
     // Init
     start = clock();
     if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3;
-    if (compressionLevel <= 3) compressionFunction = LZ4_compress_limitedOutput_local; 
+    if (compressionLevel <= 3) compressionFunction = LZ4_compress_limitedOutput_local;
     else { compressionFunction = LZ4_compressHC2_limitedOutput; }
     get_fileHandle(input_filename, output_filename, &finput, &foutput);
     blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
@@ -587,7 +612,7 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp
         outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1, compressionLevel);
         if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4;
         if (blockChecksum) compressedfilesize+=4;
-        DISPLAYLEVEL(3, "\rRead : %i MB  ==> %.2f%%   ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
+        DISPLAYLEVEL(3, "==> %.2f%%   ", (double)compressedfilesize/filesize*100);
 
         // Write Block
         if (outSize > 0)