From b077a99347a384225d03f5458c2b57bedb134c62 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 14 Jan 2018 16:57:25 -0800
Subject: added checkTag

checkTag verifies that provided tag and library version match.
It's started automatically in circleCI when a new tag is created.
---
 .travis.yml      |  8 ++++++
 tests/.gitignore |  1 +
 tests/Makefile   |  5 +++-
 tests/checkTag.c | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 tests/checkTag.c

diff --git a/.travis.yml b/.travis.yml
index 3353dee..9dea485 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -145,7 +145,15 @@ matrix:
             - gcc-multilib
             - gcc-4.4
 
+    # tag-specific test
+    - if: tag =~ ^v[0-9]\.[0-9]
+      os: linux
+      sudo: false
+      env: Cmd="make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH " COMPILER=cc
+
+
 script:
+  - uname -a
   - echo Cmd=$Cmd
   - $COMPILER -v
   - sh -c "$Cmd"
diff --git a/tests/.gitignore b/tests/.gitignore
index 4c0f311..36dff42 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -8,6 +8,7 @@ fullbench32
 fuzzer
 fuzzer32
 fasttest
+checkTag
 
 # test artefacts
 tmp*
diff --git a/tests/Makefile b/tests/Makefile
index 819ba43..ddc0d2e 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -111,7 +111,7 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
-        fasttest$(EXT) datagen$(EXT)
+        fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
 
@@ -119,6 +119,9 @@ clean:
 versionsTest:
 	$(PYTHON) test-lz4-versions.py
 
+checkTag: checkTag.c $(LZ4DIR)/lz4.h
+	$(CC) $(FLAGS) $< -o $@$(EXT)
+
 
 #-----------------------------------------------------------------------------
 # validated only for Linux, OSX, BSD, Hurd and Solaris targets
diff --git a/tests/checkTag.c b/tests/checkTag.c
new file mode 100644
index 0000000..4a33415
--- /dev/null
+++ b/tests/checkTag.c
@@ -0,0 +1,79 @@
+/*
+    checkTag.c - Version validation tool for LZ4
+    Copyright (C) Yann Collet 2018 - present
+
+    GPL v2 License
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    You can contact the author at :
+    - LZ4 homepage : http://www.lz4.org
+    - LZ4 source repo : https://github.com/lz4/lz4
+*/
+
+/* checkTag command :
+ * $ ./checkTag tag
+ * checkTag validates tags of following format : v[0-9].[0-9].[0-9]{any}
+ * The tag is then compared to LZ4 version number.
+ * They are compatible if first 3 digits are identical.
+ * Anything beyond that is free, and doesn't impact validation.
+ * Example : tag v1.8.1.2 is compatible with version 1.8.1
+ * When tag and version are not compatible, program exits with error code 1.
+ * When they are compatible, it exists with a code 0.
+ * checkTag is intended to be used in automated testing environment.
+ */
+
+#include <stdio.h>   /* printf */
+#include <string.h>  /* strlen, strncmp */
+#include "lz4.h"     /* LZ4_VERSION_STRING */
+
+
+/*  validate() :
+ * @return 1 if tag is compatible, 0 if not.
+ */
+static int validate(const char* const tag)
+{
+    size_t const tagLength = strlen(tag);
+    size_t const verLength = strlen(LZ4_VERSION_STRING);
+
+    if (tagLength < 2) return 0;
+    if (tag[0] != 'v') return 0;
+    if (tagLength <= verLength) return 0;
+
+    if (strncmp(LZ4_VERSION_STRING, tag+1, verLength)) return 0;
+
+    return 1;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+    const char* const tag = argv[1];
+    if (argc!=2) {
+        printf("incorrect usage : %s tag \n", exeName);
+        return 2;
+    }
+
+    printf("Version : %s \n", LZ4_VERSION_STRING);
+    printf("Tag     : %s \n", tag);
+
+    if (validate(tag)) {
+        printf("OK : tag is compatible with lz4 version \n");
+        return 0;
+    }
+
+    printf("!! error : tag and versions are not compatible !! \n");
+    return 1;
+}
-- 
cgit v0.12


From 7dba09af47dd3daa1562a6332a643a1a59dba4a8 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 16 Jan 2018 10:21:37 -0800
Subject: use more restrictive conditions for clock_gettime()

---
 programs/util.h | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/programs/util.h b/programs/util.h
index fc7f63e..a3576d7 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -141,6 +141,7 @@ extern "C" {
 *  Time functions
 ******************************************/
 #if defined(_WIN32)   /* Windows */
+
     typedef LARGE_INTEGER UTIL_time_t;
     UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
     UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
@@ -165,7 +166,9 @@ extern "C" {
         }
         return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
     }
+
 #elif defined(__APPLE__) && defined(__MACH__)
+
     #include <mach/mach_time.h>
     typedef U64 UTIL_time_t;
     UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
@@ -189,7 +192,9 @@ extern "C" {
         }
         return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
     }
-#elif (PLATFORM_POSIX_VERSION >= 200112L)
+
+#elif (PLATFORM_POSIX_VERSION >= 200112L) && (defined __UCLIBC__ || ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) || __GLIBC__ > 2))
+
     #include <time.h>
     typedef struct timespec UTIL_time_t;
     UTIL_STATIC UTIL_time_t UTIL_getTime(void)
@@ -227,7 +232,9 @@ extern "C" {
         nano += diff.tv_nsec;
         return nano;
     }
+
 #else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
+
     typedef clock_t UTIL_time_t;
     UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); }
     UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
-- 
cgit v0.12


From 75b81bbbf09f7d76f0e439ff29d12d922e74c0bb Mon Sep 17 00:00:00 2001
From: Po-Chuan Hsieh <sunpoet@sunpoet.net>
Date: Thu, 18 Jan 2018 03:13:05 +0800
Subject: Change file format back to ASCII (from UTF-8)

- Replace U+00A0 by space
- Fix build failure of archivers/py-borgbackup in FreeBSD

Reference: https://bugs.FreeBSD.org/225235
---
 lib/lz4.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index a06b8a4..7710b11 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -264,8 +264,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
  *  If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
  *
  *  Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
- *  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
- *  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
+ *  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
+ *  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
  *
  * @return : size of compressed block
  *           or 0 if there is an error (typically, compressed data cannot fit into 'dst')
-- 
cgit v0.12


From 30e92f320cc355c91a9b1840fd14026566ba9477 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 22 Jan 2018 12:50:06 -0800
Subject: [lz4hc] level == 0 means default, not level 1

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f2c2566..cface81 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -728,7 +728,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
-    if (compressionLevel < 1) compressionLevel = 1;
+    if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
     if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX;
     LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
 }
-- 
cgit v0.12


From fd0c125ff1c7962f6ec2567718ffa576aa4248a8 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 30 Jan 2018 17:53:00 -0800
Subject: proposed a minor change to LZ4 Frame format specification

add new terms "LZ4 Frame Header" and "LZ4 Frame Footer"
---
 doc/lz4_Frame_format.md | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md
index 77454b2..359e563 100644
--- a/doc/lz4_Frame_format.md
+++ b/doc/lz4_Frame_format.md
@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-1.6.0 (08/08/2017)
+1.6.1 (30/01/2018)
 
 
 Introduction
@@ -72,12 +72,15 @@ Value : 0x184D2204
 
 __Frame Descriptor__
 
-3 to 15 Bytes, to be detailed in the next part.
-Most important part of the spec.
+3 to 15 Bytes, to be detailed in its own paragraph,
+as it is the most important part of the spec.
+
+The combined __Magic Number__ and __Frame Descriptor__ fields are sometimes
+called ___LZ4 Frame Header___. Its size vary between 7 and 19 bytes.
 
 __Data Blocks__
 
-To be detailed later on.
+To be detailed in its own paragraph.
 That’s where compressed data is stored.
 
 __EndMark__
@@ -98,6 +101,9 @@ that all blocks were fully transmitted in the correct order and without error,
 and also that the encoding/decoding process itself generated no distortion.
 Its usage is recommended.
 
+The combined __EndMark__ and __Content Checksum__ fields might sometimes be
+referred as ___LZ4 Frame Footer___. Its size vary between 4 and 8 bytes.
+
 __Frame Concatenation__
 
 In some circumstances, it may be preferable to append multiple frames,
@@ -380,6 +386,8 @@ and trigger an error if it does not fit within acceptable range.
 Version changes
 ---------------
 
+1.6.1 : introduced terms "LZ4 Frame Header" and "LZ4 Frame Footer"
+
 1.6.0 : restored Dictionary ID field in Frame header
 
 1.5.1 : changed document format to MarkDown
-- 
cgit v0.12


From 865bd83e13590dc4a4c82f559f454cefcd86fd69 Mon Sep 17 00:00:00 2001
From: Asger Hautop Drewsen <asgerdrewsen@gmail.com>
Date: Wed, 31 Jan 2018 13:33:07 +0100
Subject: Ensure LZ4_DEPRECATED("...") is before LZ4LIB_API

When using clang++ with std c++14 or c++17 you would get the error "an attribute list cannot appear here" when including "lz4.h" as the visibility attribute is before the c++ attribute.
This ensures that the [[deprecated]] c++ attribute is before everything
else in the function declarations.
---
 lib/lz4.h   | 28 ++++++++++++++--------------
 lib/lz4hc.h | 34 +++++++++++++++++-----------------
 2 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 7710b11..df85472 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -450,26 +450,26 @@ union LZ4_streamDecode_u {
 #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
 
 /* Obsolete compression functions */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress               (const char* source, char* dest, int sourceSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState               (void* state, const char* source, char* dest, int inputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue                (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue  (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress               (const char* source, char* dest, int sourceSize);
+LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState               (void* state, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue                (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue  (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
 /* Obsolete decompression functions */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
+LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
 /* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int   LZ4_sizeofStreamState(void);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead")  int   LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead")     char* LZ4_slideInputBuffer (void* state);
+LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
+LZ4_DEPRECATED("use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
 
 /* Obsolete streaming decoding functions */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
+LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
+LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
 
 #endif /* LZ4_H_2983827168210 */
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index d41bf42..a7f77f9 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -195,25 +195,25 @@ union LZ4_streamHC_u {
 /* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */
 
 /* deprecated compression functions */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC               (const char* source, char* dest, int inputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC               (void* state, const char* source, char* dest, int inputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC               (const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_withStateHC               (void* state, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
 /* Deprecated Streaming functions using older model; should no longer be used */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDictHC() instead")     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_freeStreamHC() instead")   int   LZ4_freeHC (void* LZ4HC_Data);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int   LZ4_sizeofStreamStateHC(void);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead")  int   LZ4_resetStreamStateHC(void* state, char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
+LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API   int   LZ4_freeHC (void* LZ4HC_Data);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API int   LZ4_sizeofStreamStateHC(void);
+LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") LZ4LIB_API  int   LZ4_resetStreamStateHC(void* state, char* inputBuffer);
 
 
 #if defined (__cplusplus)
-- 
cgit v0.12


From d03224b63345d4ace45c56773f456da16e6c0acd Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 09:54:30 -0800
Subject: fix typos as suggested by @psteinb

---
 doc/lz4_Frame_format.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md
index 359e563..0c98df1 100644
--- a/doc/lz4_Frame_format.md
+++ b/doc/lz4_Frame_format.md
@@ -76,7 +76,7 @@ __Frame Descriptor__
 as it is the most important part of the spec.
 
 The combined __Magic Number__ and __Frame Descriptor__ fields are sometimes
-called ___LZ4 Frame Header___. Its size vary between 7 and 19 bytes.
+called ___LZ4 Frame Header___. Its size varies between 7 and 19 bytes.
 
 __Data Blocks__
 
@@ -102,7 +102,7 @@ and also that the encoding/decoding process itself generated no distortion.
 Its usage is recommended.
 
 The combined __EndMark__ and __Content Checksum__ fields might sometimes be
-referred as ___LZ4 Frame Footer___. Its size vary between 4 and 8 bytes.
+referred to as ___LZ4 Frame Footer___. Its size varies between 4 and 8 bytes.
 
 __Frame Concatenation__
 
-- 
cgit v0.12


From c129f480e777d795f9945bc9ae9dafe01f3772c4 Mon Sep 17 00:00:00 2001
From: Asger Hautop Drewsen <asgerdrewsen@gmail.com>
Date: Wed, 31 Jan 2018 20:23:20 +0100
Subject: Always prefer c++14 attributes if available

---
 lib/lz4.h | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index df85472..6179c05 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -433,11 +433,9 @@ union LZ4_streamDecode_u {
 #  define LZ4_DEPRECATED(message)   /* disable deprecation warnings */
 #else
 #  define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */
-#    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
-#  elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+#  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
 #    define LZ4_DEPRECATED(message) [[deprecated(message)]]
-#  elif (LZ4_GCC_VERSION >= 405)
+#  elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
 #  elif (LZ4_GCC_VERSION >= 301)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated))
-- 
cgit v0.12


From 87fb7a1d031a7e6df72d5ca50ab543a57ddca21f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 14:33:16 -0800
Subject: refactored frameCompress example

to better reflect LZ4F API usage.
---
 Makefile                 |   3 +-
 doc/lz4_manual.html      |   4 +-
 doc/lz4frame_manual.html |  44 +++++++-------
 examples/frameCompress.c | 150 ++++++++++++++++++++++++++---------------------
 lib/lz4frame.c           |   3 +-
 lib/lz4frame.h           |  12 ++--
 6 files changed, 120 insertions(+), 96 deletions(-)

diff --git a/Makefile b/Makefile
index 0aa1b9d..d17b4a7 100644
--- a/Makefile
+++ b/Makefile
@@ -70,7 +70,7 @@ lz4 lz4-release :
 
 .PHONY: examples
 examples: lib lz4
-	$(MAKE) -C $(EXDIR) test
+	$(MAKE) -C $(EXDIR) all
 
 .PHONY: manuals
 manuals:
@@ -125,6 +125,7 @@ list:
 .PHONY: test
 test:
 	$(MAKE) -C $(TESTDIR) $@
+	$(MAKE) -C $(EXDIR) $@
 
 clangtest: clean
 	clang -v
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 6b7935d..46e6c3d 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -176,8 +176,8 @@ int           LZ4_freeStream (LZ4_stream_t* streamPtr);
   If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
 
   Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
-  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
-  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
+  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
+  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
 
  @return : size of compressed block
            or 0 if there is an error (typically, compressed data cannot fit into 'dst')
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 590c632..1ddf6d8 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -101,8 +101,10 @@
 <a name="Chapter5"></a><h2>Simple compression function</h2><pre></pre>
 
 <pre><b>size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-</b><p>  Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
-  Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+</b><p>  Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
+ `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
+  Note : this result is only usable with LZ4F_compressFrame().
+         It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.
  
 </p></pre><BR>
 
@@ -151,19 +153,21 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 </p></pre><BR>
 
 <pre><b>size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
-</b><p> Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
- prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
- Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
- When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
+</b><p>  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
+  Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
+  Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
+  prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
+  Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
+  When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
  
 </p></pre><BR>
 
 <pre><b>size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
-</b><p> LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
- An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
- This value is provided by LZ4F_compressBound().
- If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
- LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
+</b><p>  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+  An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
+  This value is provided by LZ4F_compressBound().
+  If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
+  LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
  `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
            or an error code if it fails (which can be tested using LZ4F_isError())
@@ -171,8 +175,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 </p></pre><BR>
 
 <pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
-</b><p> When data must be generated and sent immediately, without waiting for a block to be completely filled,
- it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
+</b><p>  When data must be generated and sent immediately, without waiting for a block to be completely filled,
+  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
  `dstCapacity` must be large enough to ensure the operation will be successful.
  `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
  @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
@@ -200,13 +204,13 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 </b></pre><BR>
 <pre><b>LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
 LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
-</b><p> Create an LZ4F_dctx object, to track all decompression operations.
- The version provided MUST be LZ4F_VERSION.
- The function provides a pointer to an allocated and initialized LZ4F_dctx object.
- The result is an errorCode, which can be tested using LZ4F_isError().
- dctx memory can be released using LZ4F_freeDecompressionContext();
- The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
- That is, it should be == 0 if decompression has been completed fully and correctly.
+</b><p>  Create an LZ4F_dctx object, to track all decompression operations.
+  The version provided MUST be LZ4F_VERSION.
+  The function provides a pointer to an allocated and initialized LZ4F_dctx object.
+  The result is an errorCode, which can be tested using LZ4F_isError().
+  dctx memory can be released using LZ4F_freeDecompressionContext();
+  The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+  That is, it should be == 0 if decompression has been completed fully and correctly.
  
 </p></pre><BR>
 
diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index d66a8dc..35dbde2 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -1,122 +1,133 @@
-// LZ4frame API example : compress a file
-// Based on sample code from Zbigniew Jędrzejewski-Szmek
+/* LZ4frame API example : compress a file
+ * Based on sample code from Zbigniew Jędrzejewski-Szmek
+ *
+ * This example streams an input file into an output file
+ * using a bounded memory budget.
+ * Input is read in chunks of IN_CHUNK_SIZE */
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
+#include <assert.h>
 
 #include <lz4frame.h>
 
-#define BUF_SIZE 16*1024
-#define LZ4_HEADER_SIZE 19
-#define LZ4_FOOTER_SIZE 4
+
+#define IN_CHUNK_SIZE  (16*1024)
 
 static const LZ4F_preferences_t lz4_preferences = {
     { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
-      0 /* content size unknown */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
-    0,   /* compression level */
+      0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
+    0,   /* compression level; 0 == default */
     0,   /* autoflush */
     { 0, 0, 0, 0 },  /* reserved, must be set to 0 */
 };
 
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) {
-    size_t r=1;  /* function result; 1 == error, default (early exit) */
-    LZ4F_compressionContext_t ctx;
-    char *src, *buf = NULL;
-    size_t size, count_in = 0, count_out, offset = 0, frame_size;
 
+/* safe_fwrite() :
+ * performs fwrite(), ensure operation success, or immediately exit() */
+static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
+{
+    size_t const writtenSize = fwrite(buf, eltSize, nbElt, f);
+    size_t const expectedSize = eltSize * nbElt;   /* note : should check for overflow */
+    if (writtenSize < expectedSize) {
+        if (ferror(f))  /* note : ferror() must follow fwrite */
+            printf("Write failed\n");
+        else
+            printf("Short write\n");
+        exit(1);
+    }
+}
+
+
+static size_t
+compress_file(FILE* in, FILE* out,
+              unsigned long long* size_in,
+              unsigned long long* size_out)
+{
+    size_t result = 1;  /* function result; 1 == error, default (early exit) */
+    unsigned long long count_in = 0, count_out;
+
+    /* init */
+    LZ4F_compressionContext_t ctx;
     if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
-        printf("Failed to create context: error %zu\n", r);
-        return 1;
+        printf("error: failed to create context \n");
+        return result;
     }
 
-    src = malloc(BUF_SIZE);
+    char* outbuff = NULL;
+    void* const src = malloc(IN_CHUNK_SIZE);
     if (!src) {
         printf("Not enough memory\n");
         goto cleanup;
     }
 
-    frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
-    size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
-    buf = malloc(size);
-    if (!buf) {
+    size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &lz4_preferences);   /* large enough for any input <= IN_CHUNK_SIZE */
+    outbuff = malloc(outbufCapacity);
+    if (!outbuff) {
         printf("Not enough memory\n");
         goto cleanup;
     }
 
-    {   size_t const headerSize = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
+    /* write frame header */
+    assert(outbufCapacity >= LZ4F_HEADER_SIZE_MAX);
+    {   size_t const headerSize = LZ4F_compressBegin(ctx, outbuff, outbufCapacity, &lz4_preferences);
         if (LZ4F_isError(headerSize)) {
             printf("Failed to start compression: error %zu\n", headerSize);
             goto cleanup;
         }
-        offset = count_out = headerSize;
-        printf("Buffer size is %zu bytes, header size %zu bytes\n", size, headerSize);
+        count_out = headerSize;
+        printf("Buffer size is %zu bytes, header size %zu bytes\n", outbufCapacity, headerSize);
+        safe_fwrite(outbuff, 1, headerSize, out);
     }
 
-
+    /* stream file */
     for (;;) {
-        size_t const readSize = fread(src, 1, BUF_SIZE, in);
-        if (readSize == 0)
-            break;
+        size_t const readSize = fread(src, 1, outbufCapacity, in);
+        if (readSize == 0) break;
         count_in += readSize;
 
-        {   size_t const compressedSize = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, readSize, NULL);
-            if (LZ4F_isError(compressedSize)) {
-                printf("Compression failed: error %zu\n", compressedSize);
-                goto cleanup;
-            }
-            offset += compressedSize;
-            count_out += compressedSize;
+        size_t const compressedSize = LZ4F_compressUpdate(ctx,
+                                                outbuff, outbufCapacity,
+                                                src, readSize,
+                                                NULL);
+        if (LZ4F_isError(compressedSize)) {
+            printf("Compression failed: error %zu\n", compressedSize);
+            goto cleanup;
         }
 
-        if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
-            size_t writtenSize;
-            printf("Writing %zu bytes\n", offset);
-
-            writtenSize = fwrite(buf, 1, offset, out);
-            if (writtenSize < offset) {
-                if (ferror(out))  /* note : ferror() must follow fwrite */
-                    printf("Write failed\n");
-                else
-                    printf("Short write\n");
-                goto cleanup;
-            }
-
-            offset = 0;
-        }
+        printf("Writing %zu bytes\n", compressedSize);
+        safe_fwrite(outbuff, 1, compressedSize, out);
+        count_out += compressedSize;
     }
 
-    {   size_t const compressedSize = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
+    /* flush whatever remains within internal buffers */
+    {   size_t const compressedSize = LZ4F_compressEnd(ctx,
+                                            outbuff, outbufCapacity,
+                                            NULL);
         if (LZ4F_isError(compressedSize)) {
             printf("Failed to end compression: error %zu\n", compressedSize);
             goto cleanup;
         }
-        offset += compressedSize;
+
+        printf("Writing %zu bytes\n", compressedSize);
+        safe_fwrite(outbuff, 1, compressedSize, out);
         count_out += compressedSize;
     }
 
-    printf("Writing %zu bytes\n", offset);
-    {   size_t const writtenSize = fwrite(buf, 1, offset, out);
-        if (writtenSize < offset) {
-            if (ferror(out))
-                printf("Write failed\n");
-            else
-                printf("Short write\n");
-            goto cleanup;
-    }   }
-
     *size_in = count_in;
     *size_out = count_out;
-    r = 0;   /* success */
+    result = 0;   /* success */
 
  cleanup:
     LZ4F_freeCompressionContext(ctx);   /* supports free on NULL */
     free(src);
-    free(buf);
-    return r;
+    free(outbuff);
+    return result;
 }
 
+
 static size_t get_block_size(const LZ4F_frameInfo_t* info) {
     switch (info->blockSizeID) {
         case LZ4F_default:
@@ -131,7 +142,7 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) {
 }
 
 static size_t decompress_file(FILE* in, FILE* out) {
-    void* const src = malloc(BUF_SIZE);
+    void* const src = malloc(IN_CHUNK_SIZE);
     void* dst = NULL;
     size_t dstCapacity = 0;
     LZ4F_dctx* dctx = NULL;
@@ -148,7 +159,7 @@ static size_t decompress_file(FILE* in, FILE* out) {
     /* Decompression */
     while (ret != 0) {
         /* Load more input */
-        size_t srcSize = fread(src, 1, BUF_SIZE, in);
+        size_t srcSize = fread(src, 1, IN_CHUNK_SIZE, in);
         const void* srcPtr = src;
         const void* const srcEnd = srcPtr + srcSize;
         if (srcSize == 0 || ferror(in)) {
@@ -215,6 +226,7 @@ cleanup:
     return LZ4F_freeDecompressionContext(dctx);   /* note : free works on NULL */
 }
 
+
 int compare(FILE* fp0, FILE* fp1)
 {
     int result = 0;
@@ -238,6 +250,7 @@ int compare(FILE* fp0, FILE* fp1)
     return result;
 }
 
+
 int main(int argc, const char **argv) {
     char inpFilename[256] = { 0 };
     char lz4Filename[256] = { 0 };
@@ -259,8 +272,8 @@ int main(int argc, const char **argv) {
     /* compress */
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const outFp = fopen(lz4Filename, "wb");
-        size_t sizeIn = 0;
-        size_t sizeOut = 0;
+        unsigned long long sizeIn = 0;
+        unsigned long long sizeOut = 0;
         size_t ret;
 
         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
@@ -270,7 +283,8 @@ int main(int argc, const char **argv) {
             return (int)ret;
         }
         printf("%s: %zu → %zu bytes, %.1f%%\n",
-            inpFilename, sizeIn, sizeOut,
+            inpFilename,
+            (size_t)sizeIn, (size_t)sizeOut,  /* might overflow */
             (double)sizeOut / sizeIn * 100);
         printf("compress : done\n");
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 0b26f75..a394d1f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -726,7 +726,8 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
 
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
-    if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
+    if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
+        return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
     memset(&cOptionsNull, 0, sizeof(cOptionsNull));
     if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index eb55e45..15484d7 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -189,8 +189,10 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void);
 *  Simple compression function
 ***********************************/
 /*! LZ4F_compressFrameBound() :
- *  Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
- *  Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+ *  Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
+ * `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
+ *  Note : this result is only usable with LZ4F_compressFrame().
+ *         It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.
  */
 LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
 
@@ -235,7 +237,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
 /*----    Compression    ----*/
 
-#define LZ4F_HEADER_SIZE_MAX 19
+#define LZ4F_HEADER_SIZE_MAX 19   /* LZ4 Frame header size can vary from 7 to 19 bytes */
 /*! LZ4F_compressBegin() :
  *  will write the frame header into dstBuffer.
  *  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
@@ -248,7 +250,9 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
                                       const LZ4F_preferences_t* prefsPtr);
 
 /*! LZ4F_compressBound() :
- *  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case situations.
+ *  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
+ *  Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
+ *  Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
  *  prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
  *  Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
  *  When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
-- 
cgit v0.12


From 1acca240a9ce824612619dde36de44723785ebe3 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 16:11:45 -0800
Subject: ensure proper dependencies are built for /examples

also : use liblz4.a static lib to share compilation time
---
 examples/Makefile | 44 +++++++++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/examples/Makefile b/examples/Makefile
index 9321c24..c56d455 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -52,34 +52,40 @@ default: all
 all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \
      lineCompress frameCompress simpleBuffer
 
-printVersion: $(LZ4DIR)/lz4.c printVersion.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+$(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4opt.h $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h
+	$(MAKE) -C $(LZ4DIR) liblz4.a
 
-doubleBuffer: $(LZ4DIR)/lz4.c blockStreaming_doubleBuffer.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+printVersion: $(LZ4DIR)/liblz4.a printVersion.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-dictionaryRandomAccess: $(LZ4DIR)/lz4.c dictionaryRandomAccess.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+doubleBuffer: $(LZ4DIR)/liblz4.a blockStreaming_doubleBuffer.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-ringBuffer  : $(LZ4DIR)/lz4.c blockStreaming_ringBuffer.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+dictionaryRandomAccess: $(LZ4DIR)/liblz4.a dictionaryRandomAccess.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-ringBufferHC: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c HCStreaming_ringBuffer.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+ringBuffer  : $(LZ4DIR)/liblz4.a blockStreaming_ringBuffer.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-lineCompress: $(LZ4DIR)/lz4.c blockStreaming_lineByLine.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+ringBufferHC: $(LZ4DIR)/liblz4.a HCStreaming_ringBuffer.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-frameCompress: frameCompress.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT) $(LZ4DIR)/liblz4.a
+lineCompress: $(LZ4DIR)/liblz4.a blockStreaming_lineByLine.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-compressFunctions: $(LZ4DIR)/lz4.c compress_functions.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT) -lrt
+frameCompress: $(LZ4DIR)/liblz4.a frameCompress.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-simpleBuffer: $(LZ4DIR)/lz4.c simple_buffer.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+compressFunctions: $(LZ4DIR)/liblz4.a compress_functions.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT) -lrt
 
-test : all
+simpleBuffer: $(LZ4DIR)/liblz4.a simple_buffer.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
+$(LZ4) :
+	$(MAKE) -C ../programs lz4
+
+test : all $(LZ4)
 	@echo "\n=== Print Version ==="
 	./printVersion$(EXT)
 	@echo "\n=== Simple compression example ==="
-- 
cgit v0.12


From ff3c67fdb2de43cc5c27b48cf15d73e4686c74da Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 16:13:02 -0800
Subject: fixed read size, as noticed by @terrelln

---
 examples/frameCompress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 35dbde2..5d78ef0 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -84,7 +84,7 @@ compress_file(FILE* in, FILE* out,
 
     /* stream file */
     for (;;) {
-        size_t const readSize = fread(src, 1, outbufCapacity, in);
+        size_t const readSize = fread(src, 1, IN_CHUNK_SIZE, in);
         if (readSize == 0) break;
         count_in += readSize;
 
-- 
cgit v0.12


From b515ae9c9938700150bb4efba1eb78c6166293e0 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 16:39:37 -0800
Subject: refactored frameCompress.c example code

compression function returns a struct.
Also : nested structure ressources->computation
to make it easier to manage multiple exit points.
---
 examples/frameCompress.c | 132 +++++++++++++++++++++++++++++------------------
 1 file changed, 81 insertions(+), 51 deletions(-)

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 5d78ef0..5275ffe 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -16,7 +16,7 @@
 
 #define IN_CHUNK_SIZE  (16*1024)
 
-static const LZ4F_preferences_t lz4_preferences = {
+static const LZ4F_preferences_t kPrefs = {
     { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
       0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
     0,   /* compression level; 0 == default */
@@ -41,84 +41,113 @@ static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
 }
 
 
-static size_t
-compress_file(FILE* in, FILE* out,
-              unsigned long long* size_in,
-              unsigned long long* size_out)
-{
-    size_t result = 1;  /* function result; 1 == error, default (early exit) */
-    unsigned long long count_in = 0, count_out;
+/* ================================================= */
+/*     Streaming Compression example               */
+/* ================================================= */
 
-    /* init */
-    LZ4F_compressionContext_t ctx;
-    if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
-        printf("error: failed to create context \n");
-        return result;
-    }
+typedef struct {
+    int error;
+    unsigned long long size_in;
+    unsigned long long size_out;
+} compressResult_t;
 
-    char* outbuff = NULL;
-    void* const src = malloc(IN_CHUNK_SIZE);
-    if (!src) {
-        printf("Not enough memory\n");
-        goto cleanup;
-    }
+static compressResult_t
+compress_file_internal(FILE* in, FILE* out,
+                    LZ4F_compressionContext_t ctx,
+                    void* inBuff, size_t inChunkSize,
+                    void* outBuff, size_t outCapacity)
+{
+    compressResult_t result = { 1, 0, 0 };  /* result for an error */
+    unsigned long long count_in = 0, count_out;
 
-    size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &lz4_preferences);   /* large enough for any input <= IN_CHUNK_SIZE */
-    outbuff = malloc(outbufCapacity);
-    if (!outbuff) {
-        printf("Not enough memory\n");
-        goto cleanup;
-    }
+    assert(ctx != NULL);
+    assert(outCapacity >= LZ4F_HEADER_SIZE_MAX);
+    assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs));
 
     /* write frame header */
-    assert(outbufCapacity >= LZ4F_HEADER_SIZE_MAX);
-    {   size_t const headerSize = LZ4F_compressBegin(ctx, outbuff, outbufCapacity, &lz4_preferences);
+    {   size_t const headerSize = LZ4F_compressBegin(ctx, outBuff, outCapacity, &kPrefs);
         if (LZ4F_isError(headerSize)) {
             printf("Failed to start compression: error %zu\n", headerSize);
-            goto cleanup;
+            return result;
         }
         count_out = headerSize;
-        printf("Buffer size is %zu bytes, header size %zu bytes\n", outbufCapacity, headerSize);
-        safe_fwrite(outbuff, 1, headerSize, out);
+        printf("Buffer size is %zu bytes, header size %zu bytes\n", outCapacity, headerSize);
+        safe_fwrite(outBuff, 1, headerSize, out);
     }
 
     /* stream file */
     for (;;) {
-        size_t const readSize = fread(src, 1, IN_CHUNK_SIZE, in);
+        size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, in);
         if (readSize == 0) break;
         count_in += readSize;
 
         size_t const compressedSize = LZ4F_compressUpdate(ctx,
-                                                outbuff, outbufCapacity,
-                                                src, readSize,
+                                                outBuff, outCapacity,
+                                                inBuff, readSize,
                                                 NULL);
         if (LZ4F_isError(compressedSize)) {
             printf("Compression failed: error %zu\n", compressedSize);
-            goto cleanup;
+            return result;
         }
 
         printf("Writing %zu bytes\n", compressedSize);
-        safe_fwrite(outbuff, 1, compressedSize, out);
+        safe_fwrite(outBuff, 1, compressedSize, out);
         count_out += compressedSize;
     }
 
     /* flush whatever remains within internal buffers */
     {   size_t const compressedSize = LZ4F_compressEnd(ctx,
-                                            outbuff, outbufCapacity,
+                                            outBuff, outCapacity,
                                             NULL);
         if (LZ4F_isError(compressedSize)) {
             printf("Failed to end compression: error %zu\n", compressedSize);
-            goto cleanup;
+            return result;
         }
 
         printf("Writing %zu bytes\n", compressedSize);
-        safe_fwrite(outbuff, 1, compressedSize, out);
+        safe_fwrite(outBuff, 1, compressedSize, out);
         count_out += compressedSize;
     }
 
-    *size_in = count_in;
-    *size_out = count_out;
-    result = 0;   /* success */
+    result.size_in = count_in;
+    result.size_out = count_out;
+    result.error = 0;
+    return result;
+}
+
+static compressResult_t
+compress_file(FILE* in, FILE* out)
+{
+    compressResult_t result = { 1, 0, 0 };  /* == error, default (early exit) */
+
+    assert(in != NULL);
+    assert(out != NULL);
+
+    /* allocate ressources */
+    LZ4F_compressionContext_t ctx;
+    if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
+        printf("error: failed to create context \n");
+        return result;
+    }
+
+    char* outbuff = NULL;
+    void* const src = malloc(IN_CHUNK_SIZE);
+    if (!src) {
+        printf("Not enough memory\n");
+        goto cleanup;
+    }
+
+    size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs);   /* large enough for any input <= IN_CHUNK_SIZE */
+    outbuff = malloc(outbufCapacity);
+    if (!outbuff) {
+        printf("Not enough memory\n");
+        goto cleanup;
+    }
+
+    result = compress_file_internal(in, out,
+                                        ctx,
+                                        src, IN_CHUNK_SIZE,
+                                        outbuff, outbufCapacity);
 
  cleanup:
     LZ4F_freeCompressionContext(ctx);   /* supports free on NULL */
@@ -128,6 +157,10 @@ compress_file(FILE* in, FILE* out,
 }
 
 
+/* ================================================= */
+/*     Streaming decompression example               */
+/* ================================================= */
+
 static size_t get_block_size(const LZ4F_frameInfo_t* info) {
     switch (info->blockSizeID) {
         case LZ4F_default:
@@ -272,20 +305,17 @@ int main(int argc, const char **argv) {
     /* compress */
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const outFp = fopen(lz4Filename, "wb");
-        unsigned long long sizeIn = 0;
-        unsigned long long sizeOut = 0;
-        size_t ret;
 
         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
-        ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-        if (ret) {
-            printf("compress : failed with code %zu\n", ret);
-            return (int)ret;
+        compressResult_t const ret = compress_file(inpFp, outFp);
+        if (ret.error) {
+            printf("compress : failed with code %i\n", ret.error);
+            return ret.error;
         }
         printf("%s: %zu → %zu bytes, %.1f%%\n",
             inpFilename,
-            (size_t)sizeIn, (size_t)sizeOut,  /* might overflow */
-            (double)sizeOut / sizeIn * 100);
+            (size_t)ret.size_in, (size_t)ret.size_out,  /* might overflow */
+            (double)ret.size_out / ret.size_in * 100);
         printf("compress : done\n");
 
         fclose(outFp);
-- 
cgit v0.12


From 3b751a50c506224e64f39dae7e5fdc61339cf08b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 17:15:02 -0800
Subject: modified gpptest recipe

---
 Makefile | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index 0aa1b9d..786f8d4 100644
--- a/Makefile
+++ b/Makefile
@@ -159,17 +159,14 @@ platformTest: clean
 versionsTest: clean
 	$(MAKE) -C $(TESTDIR) $@
 
-gpptest: clean
-	g++ -v
-	CC=g++ $(MAKE) -C $(LZ4DIR)  all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
-	CC=g++ $(MAKE) -C $(PRGDIR)  all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
-	CC=g++ $(MAKE) -C $(TESTDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
-
-gpptest32: clean
-	g++ -v
-	CC=g++ $(MAKE) -C $(LZ4DIR)  all    CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
-	CC=g++ $(MAKE) -C $(PRGDIR)  native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
-	CC=g++ $(MAKE) -C $(TESTDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+gpptest gpptest32: CC = "$(CXX) -Wno-deprecated"
+gpptest gpptest32: CFLAGS = -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
+gpptest32: CFLAGS += -m32
+gpptest gpptest32: clean
+	$(CXX) -v
+	CC=$(CC) $(MAKE) -C $(LZ4DIR)  all CFLAGS="$(CFLAGS)"
+	CC=$(CC) $(MAKE) -C $(PRGDIR)  all CFLAGS="$(CFLAGS)"
+	CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
 
 c_standards: clean
 	# note : lz4 is not C90 compatible, because it requires long long support
-- 
cgit v0.12


From d1ccd620d62acb1863357aeb1bb64fe374b0d055 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 17:16:48 -0800
Subject: travisci : ensure "clean" betweeb 2 tests

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 9dea485..466d55e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,7 +19,7 @@ matrix:
 
     - os: linux
       sudo: false
-      env: Ubu=12.04cont Cmd="make gpptest && make clean examples && make clean cmake && make clean travis-install && make clean clangtest" COMPILER=cc
+      env: Ubu=12.04cont Cmd="make gpptest && make clean && make examples && make clean cmake && make clean travis-install && make clean clangtest" COMPILER=cc
 
 
     # 14.04 LTS Server Edition 64 bit
-- 
cgit v0.12


From 886a4858451800f6ebd621a2816a4698486d7ff1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 23:17:52 -0800
Subject: examples/Makefile : changed dependency order

static library *.a must come after source files *.c on linux
---
 Makefile          |  2 +-
 examples/Makefile | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index b48cfdd..83320fa 100644
--- a/Makefile
+++ b/Makefile
@@ -69,7 +69,7 @@ lz4 lz4-release :
 	@cp $(PRGDIR)/lz4$(EXT) .
 
 .PHONY: examples
-examples: lib lz4
+examples:
 	$(MAKE) -C $(EXDIR) all
 
 .PHONY: manuals
diff --git a/examples/Makefile b/examples/Makefile
index c56d455..f9e9e7a 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -55,31 +55,31 @@ all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \
 $(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4opt.h $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h
 	$(MAKE) -C $(LZ4DIR) liblz4.a
 
-printVersion: $(LZ4DIR)/liblz4.a printVersion.c
+printVersion: printVersion.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-doubleBuffer: $(LZ4DIR)/liblz4.a blockStreaming_doubleBuffer.c
+doubleBuffer: blockStreaming_doubleBuffer.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-dictionaryRandomAccess: $(LZ4DIR)/liblz4.a dictionaryRandomAccess.c
+dictionaryRandomAccess: dictionaryRandomAccess.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-ringBuffer  : $(LZ4DIR)/liblz4.a blockStreaming_ringBuffer.c
+ringBuffer  : blockStreaming_ringBuffer.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-ringBufferHC: $(LZ4DIR)/liblz4.a HCStreaming_ringBuffer.c
+ringBufferHC: HCStreaming_ringBuffer.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-lineCompress: $(LZ4DIR)/liblz4.a blockStreaming_lineByLine.c
+lineCompress: blockStreaming_lineByLine.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-frameCompress: $(LZ4DIR)/liblz4.a frameCompress.c
+frameCompress: frameCompress.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-compressFunctions: $(LZ4DIR)/liblz4.a compress_functions.c
+compressFunctions: compress_functions.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT) -lrt
 
-simpleBuffer: $(LZ4DIR)/liblz4.a simple_buffer.c
+simpleBuffer: simple_buffer.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 $(LZ4) :
-- 
cgit v0.12


From 25efdd80c5aee15a425f7e6bea00b4c34bfb8db2 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 1 Feb 2018 01:36:38 -0800
Subject: refactored ressource allocation to avoid goto

---
 examples/frameCompress.c | 35 ++++++++++++-----------------------
 1 file changed, 12 insertions(+), 23 deletions(-)

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 5275ffe..d62053f 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -116,40 +116,29 @@ compress_file_internal(FILE* in, FILE* out,
 }
 
 static compressResult_t
-compress_file(FILE* in, FILE* out)
+compress_file(FILE* f_in, FILE* f_out)
 {
-    compressResult_t result = { 1, 0, 0 };  /* == error, default (early exit) */
+    compressResult_t result = { 1, 0, 0 };  /* == error (default) */
 
-    assert(in != NULL);
-    assert(out != NULL);
+    assert(f_in != NULL);
+    assert(f_out != NULL);
 
-    /* allocate ressources */
+    /* ressource allocation */
     LZ4F_compressionContext_t ctx;
-    if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
-        printf("error: failed to create context \n");
-        return result;
-    }
-
-    char* outbuff = NULL;
+    size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
     void* const src = malloc(IN_CHUNK_SIZE);
-    if (!src) {
-        printf("Not enough memory\n");
-        goto cleanup;
-    }
-
     size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs);   /* large enough for any input <= IN_CHUNK_SIZE */
-    outbuff = malloc(outbufCapacity);
-    if (!outbuff) {
-        printf("Not enough memory\n");
-        goto cleanup;
-    }
+    void* const outbuff = malloc(outbufCapacity);
 
-    result = compress_file_internal(in, out,
+    if (!LZ4F_isError(ctxCreation) && src && outbuff) {
+        result = compress_file_internal(f_in, f_out,
                                         ctx,
                                         src, IN_CHUNK_SIZE,
                                         outbuff, outbufCapacity);
+    } else {
+        printf("error : ressource allocation failed \n");
+    }
 
- cleanup:
     LZ4F_freeCompressionContext(ctx);   /* supports free on NULL */
     free(src);
     free(outbuff);
-- 
cgit v0.12


From 3ce289bcce012b8173edd4c1040208da2692d060 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 1 Feb 2018 02:48:20 -0800
Subject: modified decompression part of frameCompress.c

using same logic as prior modifications for compression part.
---
 examples/frameCompress.c | 237 +++++++++++++++++++++++++++--------------------
 1 file changed, 138 insertions(+), 99 deletions(-)

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index d62053f..972f716 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -1,5 +1,5 @@
 /* LZ4frame API example : compress a file
- * Based on sample code from Zbigniew Jędrzejewski-Szmek
+ * Modified from an example code by Zbigniew Jędrzejewski-Szmek
  *
  * This example streams an input file into an output file
  * using a bounded memory budget.
@@ -52,7 +52,7 @@ typedef struct {
 } compressResult_t;
 
 static compressResult_t
-compress_file_internal(FILE* in, FILE* out,
+compress_file_internal(FILE* f_in, FILE* f_out,
                     LZ4F_compressionContext_t ctx,
                     void* inBuff, size_t inChunkSize,
                     void* outBuff, size_t outCapacity)
@@ -60,6 +60,7 @@ compress_file_internal(FILE* in, FILE* out,
     compressResult_t result = { 1, 0, 0 };  /* result for an error */
     unsigned long long count_in = 0, count_out;
 
+    assert(f_in != NULL); assert(f_out != NULL);
     assert(ctx != NULL);
     assert(outCapacity >= LZ4F_HEADER_SIZE_MAX);
     assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs));
@@ -72,13 +73,13 @@ compress_file_internal(FILE* in, FILE* out,
         }
         count_out = headerSize;
         printf("Buffer size is %zu bytes, header size %zu bytes\n", outCapacity, headerSize);
-        safe_fwrite(outBuff, 1, headerSize, out);
+        safe_fwrite(outBuff, 1, headerSize, f_out);
     }
 
     /* stream file */
     for (;;) {
-        size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, in);
-        if (readSize == 0) break;
+        size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in);
+        if (readSize == 0) break; /* nothing left to read from input file */
         count_in += readSize;
 
         size_t const compressedSize = LZ4F_compressUpdate(ctx,
@@ -91,21 +92,21 @@ compress_file_internal(FILE* in, FILE* out,
         }
 
         printf("Writing %zu bytes\n", compressedSize);
-        safe_fwrite(outBuff, 1, compressedSize, out);
+        safe_fwrite(outBuff, 1, compressedSize, f_out);
         count_out += compressedSize;
     }
 
     /* flush whatever remains within internal buffers */
     {   size_t const compressedSize = LZ4F_compressEnd(ctx,
-                                            outBuff, outCapacity,
-                                            NULL);
+                                                outBuff, outCapacity,
+                                                NULL);
         if (LZ4F_isError(compressedSize)) {
             printf("Failed to end compression: error %zu\n", compressedSize);
             return result;
         }
 
         printf("Writing %zu bytes\n", compressedSize);
-        safe_fwrite(outBuff, 1, compressedSize, out);
+        safe_fwrite(outBuff, 1, compressedSize, f_out);
         count_out += compressedSize;
     }
 
@@ -118,8 +119,6 @@ compress_file_internal(FILE* in, FILE* out,
 static compressResult_t
 compress_file(FILE* f_in, FILE* f_out)
 {
-    compressResult_t result = { 1, 0, 0 };  /* == error (default) */
-
     assert(f_in != NULL);
     assert(f_out != NULL);
 
@@ -130,6 +129,7 @@ compress_file(FILE* f_in, FILE* f_out)
     size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs);   /* large enough for any input <= IN_CHUNK_SIZE */
     void* const outbuff = malloc(outbufCapacity);
 
+    compressResult_t result = { 1, 0, 0 };  /* == error (default) */
     if (!LZ4F_isError(ctxCreation) && src && outbuff) {
         result = compress_file_internal(f_in, f_out,
                                         ctx,
@@ -158,115 +158,152 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) {
         case LZ4F_max1MB:   return 1 << 20;
         case LZ4F_max4MB:   return 1 << 22;
         default:
-            printf("Impossible unless more block sizes are allowed\n");
+            printf("Impossible with expected frame specification (<=v1.6.1)\n");
             exit(1);
     }
 }
 
-static size_t decompress_file(FILE* in, FILE* out) {
-    void* const src = malloc(IN_CHUNK_SIZE);
-    void* dst = NULL;
-    size_t dstCapacity = 0;
-    LZ4F_dctx* dctx = NULL;
+/* @return : 1==error, 0==success */
+static int
+decompress_file_internal(FILE* f_in, FILE* f_out,
+                        LZ4F_dctx* dctx,
+                        void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed,
+                        void* dst, size_t dstCapacity)
+{
+    int firstChunk = 1;
     size_t ret = 1;
 
-    /* Initialization */
-    if (!src) { perror("decompress_file(src)"); goto cleanup; }
-    {   size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100);
-        if (LZ4F_isError(dctxStatus)) {
-            printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
-            goto cleanup;
-    }   }
+    assert(f_in != NULL); assert(f_out != NULL);
+    assert(dctx != NULL);
+    assert(src != NULL); assert(srcCapacity > 0); assert(filled <= srcCapacity); assert(alreadyConsumed <= filled);
+    assert(dst != NULL); assert(dstCapacity > 0);
 
     /* Decompression */
     while (ret != 0) {
         /* Load more input */
-        size_t srcSize = fread(src, 1, IN_CHUNK_SIZE, in);
-        const void* srcPtr = src;
-        const void* const srcEnd = srcPtr + srcSize;
-        if (srcSize == 0 || ferror(in)) {
+        size_t readSize = firstChunk ? filled : fread(src, 1, srcCapacity, f_in); firstChunk=0;
+        const void* srcPtr = src + alreadyConsumed; alreadyConsumed=0;
+        const void* const srcEnd = srcPtr + readSize;
+        if (readSize == 0 || ferror(f_in)) {
             printf("Decompress: not enough input or error reading file\n");
-            goto cleanup;
-        }
-        /* Allocate destination buffer if it isn't already */
-        if (!dst) {
-            LZ4F_frameInfo_t info;
-            ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize);
-            if (LZ4F_isError(ret)) {
-                printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret));
-                goto cleanup;
-            }
-            /* Allocating enough space for an entire block isn't necessary for
-             * correctness, but it allows some memcpy's to be elided.
-             */
-            dstCapacity = get_block_size(&info);
-            dst = malloc(dstCapacity);
-            if (!dst) { perror("decompress_file(dst)"); goto cleanup; }
-            srcPtr += srcSize;
-            srcSize = srcEnd - srcPtr;
+            return 1;
         }
+
         /* Decompress:
-         * Continue while there is more input to read and the frame isn't over.
-         * If srcPtr == srcEnd then we know that there is no more output left in the
-         * internal buffer left to flush.
+         * Continue while there is more input to read (srcPtr != srcEnd)
+         * and the frame isn't over (ret != 0)
          */
         while (srcPtr != srcEnd && ret != 0) {
-            /* INVARIANT: Any data left in dst has already been written */
+            /* Any data within dst has been flushed at this stage */
             size_t dstSize = dstCapacity;
+            size_t srcSize = srcEnd - srcPtr;
             ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
             if (LZ4F_isError(ret)) {
                 printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
-                goto cleanup;
+                return 1;
             }
             /* Flush output */
-            if (dstSize != 0){
-                size_t written = fwrite(dst, 1, dstSize, out);
-                printf("Writing %zu bytes\n", dstSize);
-                if (written != dstSize) {
-                    printf("Decompress: Failed to write to file\n");
-                    goto cleanup;
-                }
-            }
+            if (dstSize != 0) safe_fwrite(dst, 1, dstSize, f_out);
             /* Update input */
             srcPtr += srcSize;
-            srcSize = srcEnd - srcPtr;
         }
     }
+
     /* Check that there isn't trailing input data after the frame.
-     * It is valid to have multiple frames in the same file, but this example
-     * doesn't support it.
+     * It is valid to have multiple frames in the same file,
+     * but this example only supports one frame.
      */
-    ret = fread(src, 1, 1, in);
-    if (ret != 0 || !feof(in)) {
-        printf("Decompress: Trailing data left in file after frame\n");
-        goto cleanup;
+    {   size_t const readSize = fread(src, 1, 1, f_in);
+        if (readSize != 0 || !feof(f_in)) {
+            printf("Decompress: Trailing data left in file after frame\n");
+            return 1;
+    }   }
+
+    return 0;
+}
+
+
+/* @return : 1==error, 0==completed */
+static int
+decompress_file_allocDst(FILE* f_in, FILE* f_out,
+                        LZ4F_dctx* dctx,
+                        void* src, size_t srcCapacity)
+{
+    assert(f_in != NULL); assert(f_out != NULL);
+    assert(dctx != NULL);
+    assert(src != NULL);
+    assert(srcCapacity >= LZ4F_HEADER_SIZE_MAX);  /* ensure LZ4F_getFrameInfo() can read enough data */
+
+    /* Read Frame header */
+    size_t const readSize = fread(src, 1, srcCapacity, f_in);
+    if (readSize == 0 || ferror(f_in)) {
+        printf("Decompress: not enough input or error reading file\n");
+        return 1;
     }
 
-cleanup:
-    free(src);
+    LZ4F_frameInfo_t info;
+    size_t consumedSize = readSize;
+    {   size_t const fires = LZ4F_getFrameInfo(dctx, &info, src, &consumedSize);
+        if (LZ4F_isError(fires)) {
+            printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(fires));
+            return 1;
+    }   }
+
+    /* Allocating enough space for an entire block isn't necessary for
+     * correctness, but it allows some memcpy's to be elided.
+     */
+    size_t const dstCapacity = get_block_size(&info);
+    void* const dst = malloc(dstCapacity);
+    if (!dst) { perror("decompress_file(dst)"); return 1; }
+
+    int const decompressionResult = decompress_file_internal(
+                        f_in, f_out,
+                        dctx,
+                        src, srcCapacity, readSize, consumedSize,
+                        dst, dstCapacity);
+
     free(dst);
-    return LZ4F_freeDecompressionContext(dctx);   /* note : free works on NULL */
+    return decompressionResult;
+}
+
+
+/* @result : 1==error, 0==success */
+static int decompress_file(FILE* f_in, FILE* f_out)
+{
+    assert(f_in != NULL); assert(f_out != NULL);
+
+    /* Ressource allocation */
+    void* const src = malloc(IN_CHUNK_SIZE);
+    if (!src) { perror("decompress_file(src)"); return 1; }
+
+    LZ4F_dctx* dctx;
+    {   size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100);
+        if (LZ4F_isError(dctxStatus)) {
+            printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
+    }   }
+
+    int const result = !dctx ? 1 /* error */ :
+                       decompress_file_allocDst(f_in, f_out, dctx, src, IN_CHUNK_SIZE);
+
+    free(src);
+    LZ4F_freeDecompressionContext(dctx);   /* note : free works on NULL */
+    return result;
 }
 
 
-int compare(FILE* fp0, FILE* fp1)
+int compareFiles(FILE* fp0, FILE* fp1)
 {
     int result = 0;
 
-    while(0 == result) {
+    while (result==0) {
         char b0[1024];
         char b1[1024];
-        const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-        const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+        size_t const r0 = fread(b0, 1, sizeof(b0), fp0);
+        size_t const r1 = fread(b1, 1, sizeof(b1), fp1);
 
-        result = (int) r0 - (int) r1;
-
-        if (0 == r0 || 0 == r1) {
-            break;
-        }
-        if (0 == result) {
-            result = memcmp(b0, b1, r0);
-        }
+        result = (r0 != r1);
+        if (!r0 || !r1) break;
+        if (!result) result = memcmp(b0, b1, r0);
     }
 
     return result;
@@ -278,7 +315,7 @@ int main(int argc, const char **argv) {
     char lz4Filename[256] = { 0 };
     char decFilename[256] = { 0 };
 
-    if(argc < 2) {
+    if (argc < 2) {
         printf("Please specify input filename\n");
         return 0;
     }
@@ -297,35 +334,36 @@ int main(int argc, const char **argv) {
 
         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
         compressResult_t const ret = compress_file(inpFp, outFp);
+
+        fclose(outFp);
+        fclose(inpFp);
+
         if (ret.error) {
             printf("compress : failed with code %i\n", ret.error);
             return ret.error;
         }
         printf("%s: %zu → %zu bytes, %.1f%%\n",
             inpFilename,
-            (size_t)ret.size_in, (size_t)ret.size_out,  /* might overflow */
+            (size_t)ret.size_in, (size_t)ret.size_out,  /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */
             (double)ret.size_out / ret.size_in * 100);
         printf("compress : done\n");
-
-        fclose(outFp);
-        fclose(inpFp);
     }
 
     /* decompress */
     {   FILE* const inpFp = fopen(lz4Filename, "rb");
         FILE* const outFp = fopen(decFilename, "wb");
-        size_t ret;
 
         printf("decompress : %s -> %s\n", lz4Filename, decFilename);
-        ret = decompress_file(inpFp, outFp);
-        if (ret) {
-            printf("decompress : failed with code %zu\n", ret);
-            return (int)ret;
-        }
-        printf("decompress : done\n");
+        int const ret = decompress_file(inpFp, outFp);
 
         fclose(outFp);
         fclose(inpFp);
+
+        if (ret) {
+            printf("decompress : failed with code %i\n", ret);
+            return ret;
+        }
+        printf("decompress : done\n");
     }
 
     /* verify */
@@ -333,15 +371,16 @@ int main(int argc, const char **argv) {
         FILE* const decFp = fopen(decFilename, "rb");
 
         printf("verify : %s <-> %s\n", inpFilename, decFilename);
-        const int cmp = compare(inpFp, decFp);
-        if(0 == cmp) {
-            printf("verify : OK\n");
-        } else {
-            printf("verify : NG\n");
-        }
+        int const cmp = compareFiles(inpFp, decFp);
 
         fclose(decFp);
         fclose(inpFp);
+
+        if (cmp) {
+            printf("corruption detected : decompressed file differs from original\n");
+            return cmp;
+        }
+        printf("verify : OK\n");
     }
 
     return 0;
-- 
cgit v0.12


From e832a3d87aeb5a8b0c250393e46c7df214ff840b Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 1 Feb 2018 16:08:59 -0800
Subject: Clarify the requirements of the LZ4 streaming API

---
 lib/lz4.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 6179c05..84f3524 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -263,7 +263,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
  *  'dst' buffer must be already allocated.
  *  If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
  *
- *  Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
+ *  Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
+ *              If less than 64KB has been compressed all the data must be present.
  *  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
  *  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
  *
@@ -306,7 +307,8 @@ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const
  *  These decoding functions allow decompression of consecutive blocks in "streaming" mode.
  *  A block is an unsplittable entity, it must be presented entirely to a decompression function.
  *  Decompression functions only accept one block at a time.
- *  Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB).
+ *  The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded.
+ *  If less than 64KB of data has been decoded all the data must be present.
  *
  *  Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
  *  - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
-- 
cgit v0.12


From 20e969e5793aa6773593df8768d068a1ae13f746 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 5 Feb 2018 15:18:00 -0800
Subject: fuzzer: added low address compression test

is expected to work on linux+gcc only.
---
 lib/lz4hc.c    |  9 +++++----
 lib/lz4hc.h    |  8 ++++----
 tests/fuzzer.c | 46 +++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index cface81..79cf651 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -856,16 +856,17 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
     LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1;   /* Error : pointer is not aligned for pointer (32 or 64 bits) */
     LZ4HC_init(ctx, (const BYTE*)inputBuffer);
-    ctx->inputBuffer = (BYTE*)inputBuffer;
+    ctx->inputBuffer = inputBuffer;
     return 0;
 }
 
-void* LZ4_createHC (char* inputBuffer)
+void* LZ4_createHC (const char* inputBuffer)
 {
     LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t));
     if (hc4 == NULL) return NULL;   /* not enough memory */
     LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
-    hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer;
+    assert(sizeof(size_t) == sizeof(void*));
+    hc4->internal_donotuse.inputBuffer = (void*)(size_t)inputBuffer;   /* ugly hack, circumvent -Wcast-qual */
     return hc4;
 }
 
@@ -889,5 +890,5 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
 {
     LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
     int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
-    return (char*)(hc4->inputBuffer + dictSize);
+    return (char*)(hc4->inputBuffer) + dictSize;
 }
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index a7f77f9..7a25bee 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -148,7 +148,7 @@ typedef struct
     const uint8_t* end;         /* next block here to continue on current prefix */
     const uint8_t* base;        /* All index relative to this position */
     const uint8_t* dictBase;    /* alternate base for extDict */
-    uint8_t* inputBuffer;       /* deprecated */
+    void*      inputBuffer;     /* deprecated */
     uint32_t   dictLimit;       /* below that point, need extDict */
     uint32_t   lowLimit;        /* below that point, no more dict */
     uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
@@ -164,7 +164,7 @@ typedef struct
     const unsigned char* end;        /* next block here to continue on current prefix */
     const unsigned char* base;       /* All index relative to this position */
     const unsigned char* dictBase;   /* alternate base for extDict */
-    unsigned char* inputBuffer;      /* deprecated */
+    void*          inputBuffer;      /* deprecated */
     unsigned int   dictLimit;        /* below that point, need extDict */
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
@@ -206,8 +206,8 @@ LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
-/* Deprecated Streaming functions using older model; should no longer be used */
-LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (char* inputBuffer);
+/* Deprecated Streaming functions; should no longer be used */
+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);
 LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
 LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API   int   LZ4_freeHC (void* LZ4HC_Data);
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c134fe3..9415e94 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -240,6 +240,42 @@ _overflowError:
 }
 
 
+#ifdef __unix__   /* is expected to be triggered on linux+gcc */
+
+#include <sys/mman.h>   /* mmap */
+
+static void* FUZ_createLowAddr(size_t size)
+{
+    void* const lowBuff = mmap((void*)(0x1000), size,
+                    PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS,
+                    -1, 0);
+    DISPLAYLEVEL(2, "generating low buffer at address %p \n", lowBuff);
+    return lowBuff;
+}
+
+static void FUZ_freeLowAddr(void* buffer, size_t size)
+{
+    if (munmap(buffer, size)) {
+        perror("fuzzer: freeing low address buffer");
+        abort();
+    }
+}
+
+#else
+
+static void* FUZ_createLowAddr(size_t size)
+{
+    return malloc(size);
+}
+
+static void FUZ_freeLowAddr(void* buffer, size_t size)
+{
+    (void)size;
+    free(buffer);
+}
+
+#endif
+
 /*! FUZ_findDiff() :
 *   find the first different byte between buff1 and buff2.
 *   presumes buff1 != buff2.
@@ -266,6 +302,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
     size_t const compressedBufferSize = LZ4_compressBound(FUZ_MAX_BLOCK_SIZE);
     char* const compressedBuffer = (char*)malloc(compressedBufferSize);
     char* const decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
+    size_t const labSize = 96 KB;
+    void* const lowAddrBuffer = FUZ_createLowAddr(labSize);
     void* const stateLZ4   = malloc(LZ4_sizeofState());
     void* const stateLZ4HC = malloc(LZ4_sizeofStateHC());
     LZ4_stream_t LZ4dict;
@@ -306,7 +344,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
         int const dictSize = MIN(dictSizeRand, blockStart);
         int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
-        char* const block = ((char*)CNBuffer) + blockStart;
+        const char* block = ((char*)CNBuffer) + blockStart;
         const char* dict = block - dictSize;
         int compressedSize, HCcompressedSize;
         int blockContinueCompressedSize;
@@ -317,6 +355,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_displayUpdate(cycleNb);
 
         /* Compression tests */
+        if ( ((FUZ_rand(&randState) & 63) == 2)
+          && ((size_t)blockSize < labSize) ) {
+            memcpy(lowAddrBuffer, block, blockSize);
+            block = lowAddrBuffer;
+        }
 
         /* Test compression destSize */
         FUZ_DISPLAYTEST;
@@ -705,6 +748,7 @@ _exit:
         free(CNBuffer);
         free(compressedBuffer);
         free(decodedBuffer);
+        FUZ_freeLowAddr(lowAddrBuffer, labSize);
         free(stateLZ4);
         free(stateLZ4HC);
         return result;
-- 
cgit v0.12


From ea25250c990769a01606dacb4149d52466b1638e Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 7 Feb 2018 02:21:25 -0800
Subject: fixed code comment as detected in #466

Also clarified a few API code comments
and updated associated html documentation
---
 doc/lz4_manual.html | 65 +++++++++++++++++++++++++++--------------------------
 lib/lz4.h           | 55 +++++++++++++++++++++++----------------------
 2 files changed, 61 insertions(+), 59 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 46e6c3d..ef715f8 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -42,9 +42,9 @@
 
 <a name="Chapter2"></a><h2>Version</h2><pre></pre>
 
-<pre><b>int LZ4_versionNumber (void);  </b>/**< library version number; to be used when checking dll version */<b>
+<pre><b>int LZ4_versionNumber (void);  </b>/**< library version number; useful to check dll version */<b>
 </b></pre><BR>
-<pre><b>const char* LZ4_versionString (void);   </b>/**< library version string; to be used when checking dll version */<b>
+<pre><b>const char* LZ4_versionString (void);   </b>/**< library version string; unseful to check dll version */<b>
 </b></pre><BR>
 <a name="Chapter3"></a><h2>Tuning parameter</h2><pre></pre>
 
@@ -53,7 +53,7 @@
 #endif
 </b><p> Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
  Increasing memory usage improves compression ratio
- Reduced memory usage can improve speed, due to cache effect
+ Reduced memory usage may improve speed, thanks to cache effect
  Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
  
 </p></pre><BR>
@@ -65,12 +65,12 @@
     into already allocated 'dst' buffer of size 'dstCapacity'.
     Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
     It also runs faster, so it's a recommended setting.
-    If the function cannot compress 'src' into a limited 'dst' budget,
+    If the function cannot compress 'src' into a more limited 'dst' budget,
     compression stops *immediately*, and the function result is zero.
-    As a consequence, 'dst' content is not valid.
-    This function never writes outside 'dst' buffer, nor read outside 'source' buffer.
-        srcSize : supported max value is LZ4_MAX_INPUT_VALUE
-        dstCapacity : full or partial size of buffer 'dst' (which must be already allocated)
+    Note : as a consequence, 'dst' content is not valid.
+    Note 2 : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
+        srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
+        dstCapacity : size of buffer 'dst' (which must be already allocated)
         return  : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
                   or 0 if compression fails 
 </p></pre><BR>
@@ -81,8 +81,7 @@
     return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
              If destination buffer is not large enough, decoding will stop and output an error code (negative value).
              If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             This function is protected against buffer overflow exploits, including malicious data packets.
-             It never writes outside output buffer, nor reads outside input buffer.
+             This function is protected against malicious data packets.
 </p></pre><BR>
 
 <a name="Chapter5"></a><h2>Advanced Functions</h2><pre></pre>
@@ -91,18 +90,18 @@
 </b><p>    Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
     This function is primarily useful for memory allocation purposes (destination buffer size).
     Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
-    Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
+    Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize)
         inputSize  : max supported value is LZ4_MAX_INPUT_SIZE
         return : maximum output size in a "worst case" scenario
-              or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
+              or 0, if input size is incorrect (too large or negative)
 </p></pre><BR>
 
 <pre><b>int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
-</b><p>    Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
+</b><p>    Same as LZ4_compress_default(), but allows selection of "acceleration" factor.
     The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
     It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
     An acceleration value of "1" is the same as regular LZ4_compress_default()
-    Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
+    Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c).
 </p></pre><BR>
 
 <pre><b>int LZ4_sizeofState(void);
@@ -125,26 +124,28 @@ int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int src
 </p></pre><BR>
 
 <pre><b>int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
-</b><p>    originalSize : is the original uncompressed size
-    return : the number of bytes read from the source buffer (in other words, the compressed size)
-             If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes.
+</b><p>This function is a bit faster than LZ4_decompress_safe(),
+but doesn't provide any security guarantee.
+    originalSize : is the uncompressed size to regenerate
+                   Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
+    return : number of bytes read from source buffer (== compressed size).
+             If the source stream is detected malformed, the function stops decoding and return a negative result.
     note : This function respects memory boundaries for *properly formed* compressed data.
-           It is a bit faster than LZ4_decompress_safe().
-           However, it does not provide any protection against intentionally modified data stream (malicious input).
-           Use this function in trusted environment only (data to decode comes from a trusted source).
+           However, it does not provide any protection against malicious input.
+           It also doesn't know 'src' size, and implies it's >= compressed size.
+           Use this function in trusted environment **only**.
 </p></pre><BR>
 
 <pre><b>int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
 </b><p>    This function decompress a compressed block of size 'srcSize' at position 'src'
     into destination buffer 'dst' of size 'dstCapacity'.
     The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that.
-    However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity).
+    However, it's not accurate, and may write more than 'targetOutputSize' (but always <= dstCapacity).
    @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity)
-       Note : this number can be < 'targetOutputSize' should the compressed block contain less data.
-             Always control how many bytes were decoded.
-             If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets.
+        Note : this number can also be < targetOutputSize, if compressed block contains less data.
+            Therefore, always control how many bytes were decoded.
+            If source stream is detected malformed, function returns a negative result.
+            This function is protected against malicious data packets.
 </p></pre><BR>
 
 <a name="Chapter6"></a><h2>Streaming Compression Functions</h2><pre></pre>
@@ -175,7 +176,8 @@ int           LZ4_freeStream (LZ4_stream_t* streamPtr);
   'dst' buffer must be already allocated.
   If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
 
-  Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
+  Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
+              If less than 64KB has been compressed all the data must be present.
   Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
   Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
 
@@ -215,7 +217,8 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
 </b><p>  These decoding functions allow decompression of consecutive blocks in "streaming" mode.
   A block is an unsplittable entity, it must be presented entirely to a decompression function.
   Decompression functions only accept one block at a time.
-  Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB).
+  The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded.
+  If less than 64KB of data has been decoded all the data must be present.
 
   Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
   - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
@@ -311,11 +314,9 @@ union LZ4_streamDecode_u {
 #  define LZ4_DEPRECATED(message)   </b>/* disable deprecation warnings */<b>
 #else
 #  define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  if defined(__clang__) </b>/* clang doesn't handle mixed C++11 and CNU attributes */<b>
-#    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
-#  elif defined (__cplusplus) && (__cplusplus >= 201402) </b>/* C++14 or greater */<b>
+#  if defined (__cplusplus) && (__cplusplus >= 201402) </b>/* C++14 or greater */<b>
 #    define LZ4_DEPRECATED(message) [[deprecated(message)]]
-#  elif (LZ4_GCC_VERSION >= 405)
+#  elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
 #  elif (LZ4_GCC_VERSION >= 301)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated))
diff --git a/lib/lz4.h b/lib/lz4.h
index 84f3524..c5d50e2 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -102,8 +102,8 @@ extern "C" {
 #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
 #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION)
 
-LZ4LIB_API int LZ4_versionNumber (void);  /**< library version number; to be used when checking dll version */
-LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string; to be used when checking dll version */
+LZ4LIB_API int LZ4_versionNumber (void);  /**< library version number; useful to check dll version */
+LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string; unseful to check dll version */
 
 
 /*-************************************
@@ -113,7 +113,7 @@ LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string;
  * LZ4_MEMORY_USAGE :
  * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
  * Increasing memory usage improves compression ratio
- * Reduced memory usage can improve speed, due to cache effect
+ * Reduced memory usage may improve speed, thanks to cache effect
  * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
  */
 #ifndef LZ4_MEMORY_USAGE
@@ -128,12 +128,12 @@ LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string;
     into already allocated 'dst' buffer of size 'dstCapacity'.
     Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
     It also runs faster, so it's a recommended setting.
-    If the function cannot compress 'src' into a limited 'dst' budget,
+    If the function cannot compress 'src' into a more limited 'dst' budget,
     compression stops *immediately*, and the function result is zero.
-    As a consequence, 'dst' content is not valid.
-    This function never writes outside 'dst' buffer, nor read outside 'source' buffer.
-        srcSize : supported max value is LZ4_MAX_INPUT_VALUE
-        dstCapacity : full or partial size of buffer 'dst' (which must be already allocated)
+    Note : as a consequence, 'dst' content is not valid.
+    Note 2 : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
+        srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
+        dstCapacity : size of buffer 'dst' (which must be already allocated)
         return  : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
                   or 0 if compression fails */
 LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
@@ -144,8 +144,7 @@ LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int
     return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
              If destination buffer is not large enough, decoding will stop and output an error code (negative value).
              If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             This function is protected against buffer overflow exploits, including malicious data packets.
-             It never writes outside output buffer, nor reads outside input buffer.
+             This function is protected against malicious data packets.
 */
 LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
 
@@ -161,20 +160,20 @@ LZ4_compressBound() :
     Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
     This function is primarily useful for memory allocation purposes (destination buffer size).
     Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
-    Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
+    Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize)
         inputSize  : max supported value is LZ4_MAX_INPUT_SIZE
         return : maximum output size in a "worst case" scenario
-              or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
+              or 0, if input size is incorrect (too large or negative)
 */
 LZ4LIB_API int LZ4_compressBound(int inputSize);
 
 /*!
 LZ4_compress_fast() :
-    Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
+    Same as LZ4_compress_default(), but allows selection of "acceleration" factor.
     The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
     It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
     An acceleration value of "1" is the same as regular LZ4_compress_default()
-    Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
+    Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c).
 */
 LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
@@ -205,15 +204,17 @@ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePt
 
 
 /*!
-LZ4_decompress_fast() : (unsafe!!)
-    originalSize : is the original uncompressed size
-    return : the number of bytes read from the source buffer (in other words, the compressed size)
-             If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes.
+LZ4_decompress_fast() : **unsafe!**
+This function is a bit faster than LZ4_decompress_safe(),
+but doesn't provide any security guarantee.
+    originalSize : is the uncompressed size to regenerate
+                   Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
+    return : number of bytes read from source buffer (== compressed size).
+             If the source stream is detected malformed, the function stops decoding and return a negative result.
     note : This function respects memory boundaries for *properly formed* compressed data.
-           It is a bit faster than LZ4_decompress_safe().
-           However, it does not provide any protection against intentionally modified data stream (malicious input).
-           Use this function in trusted environment only (data to decode comes from a trusted source).
+           However, it does not provide any protection against malicious input.
+           It also doesn't know 'src' size, and implies it's >= compressed size.
+           Use this function in trusted environment **only**.
 */
 LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
 
@@ -222,12 +223,12 @@ LZ4_decompress_safe_partial() :
     This function decompress a compressed block of size 'srcSize' at position 'src'
     into destination buffer 'dst' of size 'dstCapacity'.
     The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that.
-    However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity).
+    However, it's not accurate, and may write more than 'targetOutputSize' (but always <= dstCapacity).
    @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity)
-       Note : this number can be < 'targetOutputSize' should the compressed block contain less data.
-             Always control how many bytes were decoded.
-             If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets.
+        Note : this number can also be < targetOutputSize, if compressed block contains less data.
+            Therefore, always control how many bytes were decoded.
+            If source stream is detected malformed, function returns a negative result.
+            This function is protected against malicious data packets.
 */
 LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
 
-- 
cgit v0.12


From c4671be550fa65241c46e9e3a9214d227fbfe27f Mon Sep 17 00:00:00 2001
From: Ben Boeckel <ben.boeckel@kitware.com>
Date: Thu, 8 Feb 2018 09:15:07 -0500
Subject: intel: do not use __attribute__((packed)) on Windows

On Windows, the Intel compiler is closer to MSVC rather than GCC and
does not support the GCC attribute syntax.

Fixes #468
---
 lib/lz4.c    | 2 +-
 lib/xxhash.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 213b085..7bf8677 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -71,7 +71,7 @@
 #ifndef LZ4_FORCE_MEMORY_ACCESS   /* can be defined externally */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define LZ4_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || defined(__GNUC__)
+#  elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__)
 #    define LZ4_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
diff --git a/lib/xxhash.c b/lib/xxhash.c
index bcf1f1d..3fc97fd 100644
--- a/lib/xxhash.c
+++ b/lib/xxhash.c
@@ -52,7 +52,7 @@
 #ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define XXH_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
+#  elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \
   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 #    define XXH_FORCE_MEMORY_ACCESS 1
 #  endif
-- 
cgit v0.12


From 3ad3b0f850ed3bcf5f3faafa414f21f84706461a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 11 Feb 2018 01:42:12 -0800
Subject: slightly improved decompression speed (~+1-2%)

by making shortcut slightly more common
---
 lib/lz4.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 7bf8677..6e5e9f5 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1220,10 +1220,13 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             size_t const ll = token >> ML_BITS;
             size_t const off = LZ4_readLE16(ip+ll);
             const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */
-            if ((off >= 18) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
+            if ((off >= 8) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
                 size_t const ml = (token & ML_MASK) + MINMATCH;
                 memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/;
-                memcpy(op, matchPtr, 18); op += ml;
+                memcpy(op+ 0, matchPtr+ 0, 8);
+                memcpy(op+ 8, matchPtr+ 8, 8);
+                memcpy(op+16, matchPtr+16, 2);
+                op += ml;
                 continue;
             }
         }
-- 
cgit v0.12


From 2b674bf02f698b565b4525a36cdf4651899b88d1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 11 Feb 2018 02:45:36 -0800
Subject: slightly improved hc compression speed (+~1-2%)

by removing bad candidates faster.
---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index cface81..9f59e80 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -220,7 +220,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
         nbAttempts--;
         if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
-            if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) {
+            if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - delta + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
                     int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
     #if 0
-- 
cgit v0.12


From 219abab74bf8914d3f8761db5b398103cbbd77d7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 11 Feb 2018 22:20:09 -0800
Subject: removed LZ4_copy8

better use memcpy() directly
---
 lib/lz4.c | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 6e5e9f5..5d4bb21 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -270,11 +270,6 @@ static void LZ4_writeLE16(void* memPtr, U16 value)
     }
 }
 
-static void LZ4_copy8(void* dst, const void* src)
-{
-    memcpy(dst,src,8);
-}
-
 /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */
 LZ4_FORCE_O2_INLINE_GCC_PPC64LE
 void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
@@ -283,7 +278,7 @@ void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
     const BYTE* s = (const BYTE*)srcPtr;
     BYTE* const e = (BYTE*)dstEnd;
 
-    do { LZ4_copy8(d,s); d+=8; s+=8; } while (d<e);
+    do { memcpy(d,s,8); d+=8; s+=8; } while (d<e);
 }
 
 
@@ -1223,9 +1218,9 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             if ((off >= 8) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
                 size_t const ml = (token & ML_MASK) + MINMATCH;
                 memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/;
-                memcpy(op+ 0, matchPtr+ 0, 8);
-                memcpy(op+ 8, matchPtr+ 8, 8);
-                memcpy(op+16, matchPtr+16, 2);
+                memcpy(op + 0, matchPtr + 0, 8);
+                memcpy(op + 8, matchPtr + 8, 8);
+                memcpy(op +16, matchPtr +16, 2);
                 op += ml;
                 continue;
             }
@@ -1316,7 +1311,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             match += inc32table[offset];
             memcpy(op+4, match, 4);
             match -= dec64table[offset];
-        } else { LZ4_copy8(op, match); match+=8; }
+        } else { memcpy(op, match, 8); match+=8; }
         op += 8;
 
         if (unlikely(cpy>oend-12)) {
@@ -1329,7 +1324,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             }
             while (op<cpy) *op++ = *match++;
         } else {
-            LZ4_copy8(op, match);
+            memcpy(op, match, 8);
             if (length>16) LZ4_wildCopy(op+8, match+8, cpy);
         }
         op = cpy;   /* correction */
-- 
cgit v0.12


From d3a13397d921d9d2ebae56d0503d23c73fee39b1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 11 Feb 2018 21:03:39 -0800
Subject: slight hc speed benefit (~+1%)

by optimizing countback
---
 lib/lz4hc.c | 31 ++++++++++---------------------
 1 file changed, 10 insertions(+), 21 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 9f59e80..c888ee1 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -123,10 +123,12 @@ LZ4_FORCE_INLINE
 int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
                     const BYTE* const iMin, const BYTE* const mMin)
 {
-    int back=0;
-    while ( (ip+back > iMin)
-         && (match+back > mMin)
-         && (ip[back-1] == match[back-1]))
+    int back = 0;
+    int const min = (int)MAX(iMin - ip, mMin - match);
+    assert(ip >= iMin); assert((size_t)(ip-iMin) < (1U<<31));
+    assert(match >= mMin); assert((size_t)(match - mMin) < (1U<<31));
+    while ( (back > min)
+         && (ip[back-1] == match[back-1]) )
             back--;
     return back;
 }
@@ -223,17 +225,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
             if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - delta + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
                     int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-    #if 0
-                    /* more generic but unfortunately slower on clang */
-                    int const back = LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr);
-    #else
-                    int back = 0;
-                    while ( (ip+back > iLowLimit)
-                         && (matchPtr+back > lowPrefixPtr)
-                         && (ip[back-1] == matchPtr[back-1])) {
-                            back--;
-                    }
-    #endif
+                    int const back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
                     mlt -= back;
 
                     if (mlt > longest) {
@@ -252,10 +244,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
                 if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
                     mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
-                while ( (ip+back > iLowLimit)
-                     && (matchIndex+back > lowLimit)
-                     && (ip[back-1] == matchPtr[back-1]))
-                        back--;
+                back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictBase+lowLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {
                     longest = mlt;
@@ -333,7 +322,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     size_t length;
     BYTE* const token = (*op)++;
 
-#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2)
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6)
     static const BYTE* start = NULL;
     static U32 totalCost = 0;
     U32 const pos = (start==NULL) ? 0 : (U32)(*anchor - start);
@@ -343,7 +332,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
     if (start==NULL) start = *anchor;  /* only works for single segment */
     //g_debuglog_enable = (pos >= 2228) & (pos <= 2262);
-    DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",
+    DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",
                 pos,
                 (U32)(*ip - *anchor), matchLength, (U32)(*ip-match),
                 cost, totalCost);
-- 
cgit v0.12


From b202c672344e9676ebcadeee8506f767795611be Mon Sep 17 00:00:00 2001
From: hobomind <ak160226@yandex.ru>
Date: Wed, 14 Feb 2018 18:47:56 +0300
Subject: fix: missed semicolon at programs/lz4io.c:954

---
 programs/lz4io.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/lz4io.c b/programs/lz4io.c
index 927928a..c712fe1 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -951,7 +951,7 @@ static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigne
         total += readBytes;
         storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, storedSkips);
     }
-    if (ferror(finput)) EXM_THROW(51, "Read Error")
+    if (ferror(finput)) EXM_THROW(51, "Read Error");
 
     LZ4IO_fwriteSparseEnd(foutput, storedSkips);
     return total;
-- 
cgit v0.12


From d74f07974892aa5e8d1adc28b8ad9058e06fcadf Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 18 Feb 2018 11:00:33 -0800
Subject: update API doc regarding double-buffer strategy

answering question #473
---
 doc/lz4_manual.html | 4 +++-
 lib/lz4.h           | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index ef715f8..b5a4042 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -178,7 +178,9 @@ int           LZ4_freeStream (LZ4_stream_t* streamPtr);
 
   Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
               If less than 64KB has been compressed all the data must be present.
-  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
+  Special 1 : When input is a double-buffer, they can have any size, including < 64 KB.
+              Make sure that buffers are separated by at least one byte.
+              This way, rule becomes simple : each block depends on previous block only.
   Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
 
  @return : size of compressed block
diff --git a/lib/lz4.h b/lib/lz4.h
index c5d50e2..08f06c7 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -266,7 +266,9 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
  *
  *  Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
  *              If less than 64KB has been compressed all the data must be present.
- *  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
+ *  Special 1 : When input is a double-buffer, they can have any size, including < 64 KB.
+ *              Make sure that buffers are separated by at least one byte.
+ *              This way, rule becomes simple : each block depends on previous block only.
  *  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
  *
  * @return : size of compressed block
-- 
cgit v0.12


From 1a233c5f0fb665fa6b65c856b150bdef92654b42 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 20 Feb 2018 11:37:19 -0800
Subject: update bench.c to use less time invocations

translating into more accurate speed measurements for small sources
---
 programs/bench.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index fac2a87..91292bc 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -41,6 +41,7 @@
 #include <string.h>      /* memset */
 #include <stdio.h>       /* fprintf, fopen, ftello */
 #include <time.h>        /* clock_t, clock, CLOCKS_PER_SEC */
+#include <assert.h>      /* assert */
 
 #include "datagen.h"     /* RDG_genBuffer */
 #include "xxhash.h"
@@ -219,6 +220,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
         UTIL_time_t coolTime;
         U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 100;
+        U32 nbDecodeLoops = ((200 MB) / (srcSize+1)) + 1;  /* conservative decode speed estimate */
         U64 totalCTime=0, totalDTime=0;
         U32 cCompleted=0, dCompleted=0;
 #       define NB_MARKS 4
@@ -279,13 +281,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             /* Decompression */
             if (!dCompleted) memset(resultBuffer, 0xD6, srcSize);  /* warm result buffer */
 
-            UTIL_sleepMilli(1); /* give processor time to other processes */
+            UTIL_sleepMilli(5); /* give processor time to other processes */
             UTIL_waitForNextTick();
-            clockStart = UTIL_getTime();
 
             if (!dCompleted) {
-                U32 nbLoops = 0;
-                do {
+                U32 nbLoops;
+                clockStart = UTIL_getTime();
+                for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
                     U32 blockNb;
                     for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                         size_t const regenSize = LZ4_decompress_safe(blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize);
@@ -294,15 +296,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                             clockLoop = 0;   /* force immediate test end */
                             break;
                         }
-
                         blockTable[blockNb].resSize = regenSize;
-                    }
-                    nbLoops++;
-                } while (UTIL_clockSpanMicro(clockStart) < DECOMP_MULT*clockLoop);
+                }   }
                 {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
                     if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
+                    nbDecodeLoops = (U32)(1000000/*1sec*/ / fastestD) + 1;  /* aim for ~1sec */
                     totalDTime += clockSpan;
-                    dCompleted = totalDTime>(DECOMP_MULT*maxTime);
+                    dCompleted = totalDTime > (DECOMP_MULT*maxTime);
             }   }
 
             markNb = (markNb+1) % NB_MARKS;
-- 
cgit v0.12


From ae3dededed1c6c1a9b4c631d9edba690c4abb59c Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 20 Feb 2018 13:05:22 -0800
Subject: ensure bench speed measurement is more accurate for small inputs

Previous method would produce too many time() invocations,
becoming a significant fraction of workload measured.

The new strategy is to use time() only once per batch,
and dynamically resize batch size so that each round lasts approximately 1 second.

This only matters for small inputs.
Measurement for large files (such as silesia.tar) are much less impacted
(though decoding speed is so fast that even medium-size files will notice an improvement).
---
 programs/bench.c | 57 +++++++++++++++++++++++++++++++++-----------------------
 programs/util.h  |  8 +++++++-
 2 files changed, 41 insertions(+), 24 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 91292bc..0b1cf5c 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -67,6 +67,7 @@ static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSi
 
 #define NBSECONDS             3
 #define TIMELOOP_MICROSEC     1*1000000ULL /* 1 second */
+#define TIMELOOP_NANOSEC      1*1000000000ULL /* 1 second */
 #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
 #define COOLPERIOD_SEC        10
 #define DECOMP_MULT           1 /* test decompression DECOMP_MULT times longer than compression */
@@ -219,8 +220,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
     {   U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
         U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
         UTIL_time_t coolTime;
-        U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 100;
-        U32 nbDecodeLoops = ((200 MB) / (srcSize+1)) + 1;  /* conservative decode speed estimate */
+        U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 100;
+        U32 nbCompressionLoops = ((5 MB) / (srcSize+1)) + 1;  /* conservative initial compression speed estimate */
+        U32 nbDecodeLoops = ((200 MB) / (srcSize+1)) + 1;  /* conservative initial decode speed estimate */
         U64 totalCTime=0, totalDTime=0;
         U32 cCompleted=0, dCompleted=0;
 #       define NB_MARKS 4
@@ -232,9 +234,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         coolTime = UTIL_getTime();
         DISPLAYLEVEL(2, "\r%79s\r", "");
         while (!cCompleted || !dCompleted) {
-            UTIL_time_t clockStart;
-            U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
-
             /* overheat protection */
             if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) {
                 DISPLAYLEVEL(2, "\rcooling down ...    \r");
@@ -248,21 +247,27 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
 
             UTIL_sleepMilli(1);  /* give processor time to other processes */
             UTIL_waitForNextTick();
-            clockStart = UTIL_getTime();
 
             if (!cCompleted) {   /* still some time to do compression tests */
-                U32 nbLoops = 0;
-                do {
+                UTIL_time_t const clockStart = UTIL_getTime();
+                U32 nbLoops;
+                for (nbLoops=0; nbLoops < nbCompressionLoops; nbLoops++) {
                     U32 blockNb;
                     for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                         size_t const rSize = compP.compressionFunction(blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr, (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom, cLevel);
                         if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4_compress() failed");
                         blockTable[blockNb].cSize = rSize;
+                }   }
+                {   U64 const clockSpan = UTIL_clockSpanNano(clockStart);
+                    if (clockSpan > 0) {
+                        if (clockSpan < fastestC * nbCompressionLoops)
+                            fastestC = clockSpan / nbCompressionLoops;
+                        assert(fastestC > 0);
+                        nbCompressionLoops = (U32)(1000000000/*1sec*/ / fastestC) + 1;  /* aim for ~1sec */
+                    } else {
+                        assert(nbCompressionLoops < 40000000);   /* avoid overflow */
+                        nbCompressionLoops *= 100;
                     }
-                    nbLoops++;
-                } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
-                {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
-                    if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
                     totalCTime += clockSpan;
                     cCompleted = totalCTime>maxTime;
             }   }
@@ -274,7 +279,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             markNb = (markNb+1) % NB_MARKS;
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
                     marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
-                    (double)srcSize / fastestC );
+                    ((double)srcSize / fastestC) * 1000 );
 
             (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
 #if 1
@@ -285,22 +290,28 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             UTIL_waitForNextTick();
 
             if (!dCompleted) {
+                UTIL_time_t const clockStart = UTIL_getTime();
                 U32 nbLoops;
-                clockStart = UTIL_getTime();
                 for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
                     U32 blockNb;
                     for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                         size_t const regenSize = LZ4_decompress_safe(blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize);
                         if (LZ4_isError(regenSize)) {
-                            DISPLAY("LZ4_decompress_safe() failed on block %u  \n", blockNb);
-                            clockLoop = 0;   /* force immediate test end */
+                            DISPLAY("LZ4_decompress_safe() failed on block %u \n", blockNb);
                             break;
                         }
                         blockTable[blockNb].resSize = regenSize;
                 }   }
-                {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
-                    if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
-                    nbDecodeLoops = (U32)(1000000/*1sec*/ / fastestD) + 1;  /* aim for ~1sec */
+                {   U64 const clockSpan = UTIL_clockSpanNano(clockStart);
+                    if (clockSpan > 0) {
+                        if (clockSpan < fastestD * nbDecodeLoops)
+                            fastestD = clockSpan / nbDecodeLoops;
+                        assert(fastestD > 0);
+                        nbDecodeLoops = (U32)(1000000000/*1sec*/ / fastestD) + 1;  /* aim for ~1sec */
+                    } else {
+                        assert(nbDecodeLoops < 40000000);   /* avoid overflow */
+                        nbDecodeLoops *= 100;
+                    }
                     totalDTime += clockSpan;
                     dCompleted = totalDTime > (DECOMP_MULT*maxTime);
             }   }
@@ -308,8 +319,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             markNb = (markNb+1) % NB_MARKS;
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
                     marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
-                    (double)srcSize / fastestC,
-                    (double)srcSize / fastestD );
+                    ((double)srcSize / fastestC) * 1000,
+                    ((double)srcSize / fastestD) * 1000);
 
             /* CRC Checking */
             {   U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
@@ -339,8 +350,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         }   /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
 
         if (g_displayLevel == 1) {
-            double cSpeed = (double)srcSize / fastestC;
-            double dSpeed = (double)srcSize / fastestD;
+            double const cSpeed = ((double)srcSize / fastestC) * 1000;
+            double const dSpeed = ((double)srcSize / fastestD) * 1000;
             if (g_additionalParam)
                 DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
             else
diff --git a/programs/util.h b/programs/util.h
index a3576d7..ff25106 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -180,7 +180,7 @@ extern "C" {
             mach_timebase_info(&rate);
             init = 1;
         }
-        return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
+        return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom)) / 1000ULL;
     }
     UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
     {
@@ -249,6 +249,12 @@ UTIL_STATIC U64 UTIL_clockSpanMicro(UTIL_time_t clockStart)
     return UTIL_getSpanTimeMicro(clockStart, clockEnd);
 }
 
+/* returns time span in nanoseconds */
+UTIL_STATIC U64 UTIL_clockSpanNano(UTIL_time_t clockStart)
+{
+    UTIL_time_t const clockEnd = UTIL_getTime();
+    return UTIL_getSpanTimeNano(clockStart, clockEnd);
+}
 
 UTIL_STATIC void UTIL_waitForNextTick(void)
 {
-- 
cgit v0.12


From 34c1634d4bb8058909a736c284e889e5c0d7f146 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 20 Feb 2018 14:13:13 -0800
Subject: fixed minor conversion warning

---
 programs/bench.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 0b1cf5c..14b6dd1 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -221,8 +221,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
         UTIL_time_t coolTime;
         U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 100;
-        U32 nbCompressionLoops = ((5 MB) / (srcSize+1)) + 1;  /* conservative initial compression speed estimate */
-        U32 nbDecodeLoops = ((200 MB) / (srcSize+1)) + 1;  /* conservative initial decode speed estimate */
+        U32 nbCompressionLoops = (U32)((5 MB) / (srcSize+1)) + 1;  /* conservative initial compression speed estimate */
+        U32 nbDecodeLoops = (U32)((200 MB) / (srcSize+1)) + 1;  /* conservative initial decode speed estimate */
         U64 totalCTime=0, totalDTime=0;
         U32 cCompleted=0, dCompleted=0;
 #       define NB_MARKS 4
-- 
cgit v0.12


From 25b16e8a2e51c41f5864ce5cc94ce65ab6fd52b9 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 20 Feb 2018 15:25:45 -0800
Subject: added one assert()

suggested by @terrelln
---
 lib/lz4hc.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c888ee1..f3631c5 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -135,7 +135,8 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
 
 /* LZ4HC_countPattern() :
  * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */
-static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32)
+static unsigned
+LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32)
 {
     const BYTE* const iStart = ip;
     reg_t const pattern = (sizeof(pattern)==8) ? (reg_t)pattern32 + (((reg_t)pattern32) << 32) : pattern32;
@@ -167,7 +168,8 @@ static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 c
 /* LZ4HC_reverseCountPattern() :
  * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!)
  * read using natural platform endianess */
-static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
+static unsigned
+LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
 {
     const BYTE* const iStart = ip;
 
@@ -185,7 +187,8 @@ static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow
 
 typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e;
 
-LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
+LZ4_FORCE_INLINE int
+LZ4HC_InsertAndGetWiderMatch (
     LZ4HC_CCtx_internal* hc4,
     const BYTE* const ip,
     const BYTE* const iLowLimit,
@@ -222,6 +225,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
         nbAttempts--;
         if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
+            assert(longest >= 1);
             if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - delta + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
                     int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-- 
cgit v0.12


From 179670f32f4960f00a89c7ef08382b06dd024873 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 20 Feb 2018 15:26:59 -0800
Subject: use TIMELOOP_NANOSEC,

as suggested by @terrelln
---
 programs/bench.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 14b6dd1..002eac9 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -263,7 +263,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         if (clockSpan < fastestC * nbCompressionLoops)
                             fastestC = clockSpan / nbCompressionLoops;
                         assert(fastestC > 0);
-                        nbCompressionLoops = (U32)(1000000000/*1sec*/ / fastestC) + 1;  /* aim for ~1sec */
+                        nbCompressionLoops = (U32)(TIMELOOP_NANOSEC / fastestC) + 1;  /* aim for ~1sec */
                     } else {
                         assert(nbCompressionLoops < 40000000);   /* avoid overflow */
                         nbCompressionLoops *= 100;
@@ -307,7 +307,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         if (clockSpan < fastestD * nbDecodeLoops)
                             fastestD = clockSpan / nbDecodeLoops;
                         assert(fastestD > 0);
-                        nbDecodeLoops = (U32)(1000000000/*1sec*/ / fastestD) + 1;  /* aim for ~1sec */
+                        nbDecodeLoops = (U32)(TIMELOOP_NANOSEC / fastestD) + 1;  /* aim for ~1sec */
                     } else {
                         assert(nbDecodeLoops < 40000000);   /* avoid overflow */
                         nbDecodeLoops *= 100;
-- 
cgit v0.12


From 7173a631db61ab9535bd0d6e5e00e9dc081d4df3 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 24 Feb 2018 11:47:53 -0800
Subject: edge case : compress up to end-mflimit (12 bytes)

The LZ4 block format specification
states that the last match must start
at a minimum distance of 12 bytes from the end of the block.

However, out of an abundance of caution,
the reference implementation would actually stop searching matches
at 13 bytes from the end of the block.

This patch fixes this small detail.
The new version is now able to properly compress a limit case
such as `aaaaaaaabaaa\n`
as reported by Gao Xiang (@hsiangkao).

Obviously, it doesn't change a lot of things.
This is just one additional match candidate per block, with a maximum match length of 7 (since last 5 bytes must remain literals).

With default policy, blocks are 4 MB long, so it doesn't happen too often
Compressing silesia.tar at default level 1 saves 5 bytes (100930101 -> 100930096).
At max level 12, it saves a grand 16 bytes (77389871 -> 77389855).

The impact is a bit more visible when blocks are smaller, hence more numerous.
For example, compressing silesia with blocks of 64 KB (using -12 -B4D) saves 543 bytes (77304583 -> 77304040).
So the smaller the packet size, the more visible the impact.

And it happens we have a ton of scenarios with little blocks using LZ4 compression ...

And a useless "hooray" sidenote :
the patch improves the LZ4 compression record of silesia (using -12 -B7D --no-frame-crc) by 16 bytes (77270672 -> 77270656)
and the record on enwik9 by 44 bytes (371680396 -> 371680352) (previously claimed by [smallz4](http://create.stephan-brumme.com/smallz4/) ).
---
 lib/lz4.c    | 7 ++++---
 lib/lz4hc.c  | 6 +++---
 lib/lz4opt.h | 6 +++---
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 5d4bb21..38a865f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -545,7 +545,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source;
     const BYTE* anchor = (const BYTE*) source;
     const BYTE* const iend = ip + inputSize;
-    const BYTE* const mflimit = iend - MFLIMIT;
+    const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1;
     const BYTE* const matchlimit = iend - LASTLITERALS;
 
     BYTE* op = (BYTE*) dest;
@@ -594,7 +594,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardIp += step;
                 step = (searchMatchNb++ >> LZ4_skipTrigger);
 
-                if (unlikely(forwardIp > mflimit)) goto _last_literals;
+                if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
+                assert(ip < mflimitPlusOne);
 
                 match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
                 if (dict==usingExtDict) {
@@ -680,7 +681,7 @@ _next_match:
         anchor = ip;
 
         /* Test end of chunk */
-        if (ip > mflimit) break;
+        if (ip >= mflimitPlusOne) break;
 
         /* Fill table */
         LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f3631c5..726cfaa 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -425,7 +425,7 @@ static int LZ4HC_compress_hashChain (
     if (inputSize < LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* Main Loop */
-    while (ip < mflimit) {
+    while (ip <= mflimit) {
         ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis);
         if (ml<MINMATCH) { ip++; continue; }
 
@@ -435,7 +435,7 @@ static int LZ4HC_compress_hashChain (
         ml0 = ml;
 
 _Search2:
-        if (ip+ml < mflimit)
+        if (ip+ml <= mflimit)
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
                             maxNbAttempts, patternAnalysis);
@@ -482,7 +482,7 @@ _Search3:
         }
         /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
 
-        if (start2 + ml2 < mflimit)
+        if (start2 + ml2 <= mflimit)
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
                             maxNbAttempts, patternAnalysis);
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 5a8438c..9bd4822 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -127,7 +127,7 @@ static int LZ4HC_compress_optimal (
 
     /* Main Loop */
     assert(ip - anchor < LZ4_MAX_INPUT_SIZE);
-    while (ip < mflimit) {
+    while (ip <= mflimit) {
         int const llen = (int)(ip - anchor);
         int best_mlen, best_off;
         int cur, last_match_pos = 0;
@@ -186,7 +186,7 @@ static int LZ4HC_compress_optimal (
             const BYTE* const curPtr = ip + cur;
             LZ4HC_match_t newMatch;
 
-            if (curPtr >= mflimit) break;
+            if (curPtr > mflimit) break;
             DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
                     cur, opt[cur].price, opt[cur+1].price, cur+1);
             if (fullUpdate) {
@@ -314,7 +314,7 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
                 if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) )   /* updates ip, op and anchor */
                     goto _dest_overflow;
         }   }
-    }  /* while (ip < mflimit) */
+    }  /* while (ip <= mflimit) */
 
 _last_literals:
     /* Encode Last Literals */
-- 
cgit v0.12


From 550b40849f6b77850453a1190b97430b121802bc Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 25 Feb 2018 00:32:09 -0800
Subject: merge lz4opt.h into lz4hc.c

Having a dedicated file for optimal parser
made sense during its creation,
it allowed Przemyslaw to work more freely on lz4opt, with less dependency on lz4hc,
moreover, the optimal parser was more complex, with its own search functions.

Since the optimal was rewritten last year, it's now a lot lighter.
It makes more sense now to integrate it directly inside lz4hc.c,
making it easier to edit (editors are a bit "lost" inside a `*.h` dependent on its #include position),
it also reduces the number of files in the project,
which fits pretty well with lz4 objectives.
(adding lz4hc requires "just" lz4hc.h and lz4hc.c).
---
 examples/Makefile |   2 +-
 lib/README.md     |   2 +-
 lib/lz4hc.c       | 331 +++++++++++++++++++++++++++++++++++++++++++++++++-
 lib/lz4opt.h      | 356 ------------------------------------------------------
 4 files changed, 329 insertions(+), 362 deletions(-)
 delete mode 100644 lib/lz4opt.h

diff --git a/examples/Makefile b/examples/Makefile
index f9e9e7a..103e7ec 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -52,7 +52,7 @@ default: all
 all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \
      lineCompress frameCompress simpleBuffer
 
-$(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4opt.h $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h
+$(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h
 	$(MAKE) -C $(LZ4DIR) liblz4.a
 
 printVersion: printVersion.c $(LZ4DIR)/liblz4.a
diff --git a/lib/README.md b/lib/README.md
index fc5d4e9..7082fe3 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -15,7 +15,7 @@ They generate and decode data using [LZ4 block format].
 
 For more compression ratio at the cost of compression speed,
 the High Compression variant called **lz4hc** is available.
-Add files **`lz4hc.c`**, **`lz4hc.h`** and **`lz4opt.h`**.
+Add files **`lz4hc.c`** and **`lz4hc.h`**.
 The variant still depends on regular `lib/lz4.*` source files.
 
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 726cfaa..2a6e080 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -67,6 +67,7 @@
 
 /*===   Constants   ===*/
 #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
+#define LZ4_OPT_NUM   (1<<12)
 
 
 /*===   Macros   ===*/
@@ -383,10 +384,6 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     return 0;
 }
 
-/* btopt */
-#include "lz4opt.h"
-
-
 static int LZ4HC_compress_hashChain (
     LZ4HC_CCtx_internal* const ctx,
     const char* const source,
@@ -605,6 +602,12 @@ _dest_overflow:
     return 0;
 }
 
+static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx,
+    const char* const source, char* dst,
+    int* srcSizePtr, int dstCapacity,
+    int const nbSearches, size_t sufficient_len,
+    limitedOutput_directive limit, int const fullUpdate);
+
 
 static int LZ4HC_compress_generic (
     LZ4HC_CCtx_internal* const ctx,
@@ -884,3 +887,323 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
     int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
     return (char*)(hc4->inputBuffer + dictSize);
 }
+
+
+/* ================================================
+ * LZ4 Optimal parser (levels 10-12)
+ * ===============================================*/
+typedef struct {
+    int price;
+    int off;
+    int mlen;
+    int litlen;
+} LZ4HC_optimal_t;
+
+/* price in bytes */
+LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen)
+{
+    int price = litlen;
+    if (litlen >= (int)RUN_MASK)
+        price += 1 + (litlen-RUN_MASK)/255;
+    return price;
+}
+
+
+/* requires mlen >= MINMATCH */
+LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
+{
+    int price = 1 + 2 ; /* token + 16-bit offset */
+
+    price += LZ4HC_literalsPrice(litlen);
+
+    if (mlen >= (int)(ML_MASK+MINMATCH))
+        price += 1 + (mlen-(ML_MASK+MINMATCH))/255;
+
+    return price;
+}
+
+
+typedef struct {
+    int off;
+    int len;
+} LZ4HC_match_t;
+
+LZ4_FORCE_INLINE LZ4HC_match_t
+LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
+                      const BYTE* ip, const BYTE* const iHighLimit,
+                      int minLen, int nbSearches)
+{
+    LZ4HC_match_t match = { 0 , 0 };
+    const BYTE* matchPtr = NULL;
+    /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
+     * but this won't be the case here, as we define iLowLimit==ip,
+     * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
+    int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
+                                ip, ip, iHighLimit, minLen, &matchPtr, &ip,
+                                nbSearches, 1 /* patternAnalysis */);
+    if (matchLength <= minLen) return match;
+    match.len = matchLength;
+    match.off = (int)(ip-matchPtr);
+    return match;
+}
+
+
+static int LZ4HC_compress_optimal (
+    LZ4HC_CCtx_internal* ctx,
+    const char* const source,
+    char* dst,
+    int* srcSizePtr,
+    int dstCapacity,
+    int const nbSearches,
+    size_t sufficient_len,
+    limitedOutput_directive limit,
+    int const fullUpdate
+    )
+{
+#define TRAILING_LITERALS 3
+    LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS];   /* ~64 KB, which is a bit large for stack... */
+
+    const BYTE* ip = (const BYTE*) source;
+    const BYTE* anchor = ip;
+    const BYTE* const iend = ip + *srcSizePtr;
+    const BYTE* const mflimit = iend - MFLIMIT;
+    const BYTE* const matchlimit = iend - LASTLITERALS;
+    BYTE* op = (BYTE*) dst;
+    BYTE* opSaved = (BYTE*) dst;
+    BYTE* oend = op + dstCapacity;
+
+    /* init */
+    DEBUGLOG(5, "LZ4HC_compress_optimal");
+    *srcSizePtr = 0;
+    if (limit == limitedDestSize) oend -= LASTLITERALS;   /* Hack for support LZ4 format restriction */
+    if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
+
+    /* Main Loop */
+    assert(ip - anchor < LZ4_MAX_INPUT_SIZE);
+    while (ip <= mflimit) {
+         int const llen = (int)(ip - anchor);
+         int best_mlen, best_off;
+         int cur, last_match_pos = 0;
+
+         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
+         if (firstMatch.len==0) { ip++; continue; }
+
+         if ((size_t)firstMatch.len > sufficient_len) {
+             /* good enough solution : immediate encoding */
+             int const firstML = firstMatch.len;
+             const BYTE* const matchPos = ip - firstMatch.off;
+             opSaved = op;
+             if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) )   /* updates ip, op and anchor */
+                 goto _dest_overflow;
+             continue;
+         }
+
+         /* set prices for first positions (literals) */
+         {   int rPos;
+             for (rPos = 0 ; rPos < MINMATCH ; rPos++) {
+                 int const cost = LZ4HC_literalsPrice(llen + rPos);
+                 opt[rPos].mlen = 1;
+                 opt[rPos].off = 0;
+                 opt[rPos].litlen = llen + rPos;
+                 opt[rPos].price = cost;
+                 DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
+                             rPos, cost, opt[rPos].litlen);
+         }   }
+         /* set prices using initial match */
+         {   int mlen = MINMATCH;
+             int const matchML = firstMatch.len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
+             int const offset = firstMatch.off;
+             assert(matchML < LZ4_OPT_NUM);
+             for ( ; mlen <= matchML ; mlen++) {
+                 int const cost = LZ4HC_sequencePrice(llen, mlen);
+                 opt[mlen].mlen = mlen;
+                 opt[mlen].off = offset;
+                 opt[mlen].litlen = llen;
+                 opt[mlen].price = cost;
+                 DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup",
+                             mlen, cost, mlen);
+         }   }
+         last_match_pos = firstMatch.len;
+         {   int addLit;
+             for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
+                 opt[last_match_pos+addLit].mlen = 1; /* literal */
+                 opt[last_match_pos+addLit].off = 0;
+                 opt[last_match_pos+addLit].litlen = addLit;
+                 opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
+                 DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
+                             last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
+         }   }
+
+         /* check further positions */
+         for (cur = 1; cur < last_match_pos; cur++) {
+             const BYTE* const curPtr = ip + cur;
+             LZ4HC_match_t newMatch;
+
+             if (curPtr > mflimit) break;
+             DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
+                     cur, opt[cur].price, opt[cur+1].price, cur+1);
+             if (fullUpdate) {
+                 /* not useful to search here if next position has same (or lower) cost */
+                 if ( (opt[cur+1].price <= opt[cur].price)
+                   /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */
+                   && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) )
+                     continue;
+             } else {
+                 /* not useful to search here if next position has same (or lower) cost */
+                 if (opt[cur+1].price <= opt[cur].price) continue;
+             }
+
+             DEBUGLOG(7, "search at rPos:%u", cur);
+             if (fullUpdate)
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
+             else
+                 /* only test matches of minimum length; slightly faster, but misses a few bytes */
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
+             if (!newMatch.len) continue;
+
+             if ( ((size_t)newMatch.len > sufficient_len)
+               || (newMatch.len + cur >= LZ4_OPT_NUM) ) {
+                 /* immediate encoding */
+                 best_mlen = newMatch.len;
+                 best_off = newMatch.off;
+                 last_match_pos = cur + 1;
+                 goto encode;
+             }
+
+             /* before match : set price with literals at beginning */
+             {   int const baseLitlen = opt[cur].litlen;
+                 int litlen;
+                 for (litlen = 1; litlen < MINMATCH; litlen++) {
+                     int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen);
+                     int const pos = cur + litlen;
+                     if (price < opt[pos].price) {
+                         opt[pos].mlen = 1; /* literal */
+                         opt[pos].off = 0;
+                         opt[pos].litlen = baseLitlen+litlen;
+                         opt[pos].price = price;
+                         DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)",
+                                     pos, price, opt[pos].litlen);
+             }   }   }
+
+             /* set prices using match at position = cur */
+             {   int const matchML = newMatch.len;
+                 int ml = MINMATCH;
+
+                 assert(cur + newMatch.len < LZ4_OPT_NUM);
+                 for ( ; ml <= matchML ; ml++) {
+                     int const pos = cur + ml;
+                     int const offset = newMatch.off;
+                     int price;
+                     int ll;
+                     DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)",
+                                 pos, last_match_pos);
+                     if (opt[cur].mlen == 1) {
+                         ll = opt[cur].litlen;
+                         price = ((cur > ll) ? opt[cur - ll].price : 0)
+                               + LZ4HC_sequencePrice(ll, ml);
+                     } else {
+                         ll = 0;
+                         price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
+                     }
+
+                     if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) {
+                         DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
+                                     pos, price, ml);
+                         assert(pos < LZ4_OPT_NUM);
+                         if ( (ml == matchML)  /* last pos of last match */
+                           && (last_match_pos < pos) )
+                             last_match_pos = pos;
+                         opt[pos].mlen = ml;
+                         opt[pos].off = offset;
+                         opt[pos].litlen = ll;
+                         opt[pos].price = price;
+             }   }   }
+             /* complete following positions with literals */
+             {   int addLit;
+                 for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
+                     opt[last_match_pos+addLit].mlen = 1; /* literal */
+                     opt[last_match_pos+addLit].off = 0;
+                     opt[last_match_pos+addLit].litlen = addLit;
+                     opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
+                     DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
+             }   }
+         }  /* for (cur = 1; cur <= last_match_pos; cur++) */
+
+         best_mlen = opt[last_match_pos].mlen;
+         best_off = opt[last_match_pos].off;
+         cur = last_match_pos - best_mlen;
+
+ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
+         assert(cur < LZ4_OPT_NUM);
+         assert(last_match_pos >= 1);  /* == 1 when only one candidate */
+         DEBUGLOG(6, "reverse traversal, looking for shortest path")
+         DEBUGLOG(6, "last_match_pos = %i", last_match_pos);
+         {   int candidate_pos = cur;
+             int selected_matchLength = best_mlen;
+             int selected_offset = best_off;
+             while (1) {  /* from end to beginning */
+                 int const next_matchLength = opt[candidate_pos].mlen;  /* can be 1, means literal */
+                 int const next_offset = opt[candidate_pos].off;
+                 DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
+                 opt[candidate_pos].mlen = selected_matchLength;
+                 opt[candidate_pos].off = selected_offset;
+                 selected_matchLength = next_matchLength;
+                 selected_offset = next_offset;
+                 if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */
+                 assert(next_matchLength > 0);  /* can be 1, means literal */
+                 candidate_pos -= next_matchLength;
+         }   }
+
+         /* encode all recorded sequences in order */
+         {   int rPos = 0;  /* relative position (to ip) */
+             while (rPos < last_match_pos) {
+                 int const ml = opt[rPos].mlen;
+                 int const offset = opt[rPos].off;
+                 if (ml == 1) { ip++; rPos++; continue; }  /* literal; note: can end up with several literals, in which case, skip them */
+                 rPos += ml;
+                 assert(ml >= MINMATCH);
+                 assert((offset >= 1) && (offset <= MAX_DISTANCE));
+                 opSaved = op;
+                 if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) )   /* updates ip, op and anchor */
+                     goto _dest_overflow;
+         }   }
+     }  /* while (ip <= mflimit) */
+
+ _last_literals:
+     /* Encode Last Literals */
+     {   size_t lastRunSize = (size_t)(iend - anchor);  /* literals */
+         size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
+         size_t const totalSize = 1 + litLength + lastRunSize;
+         if (limit == limitedDestSize) oend += LASTLITERALS;  /* restore correct value */
+         if (limit && (op + totalSize > oend)) {
+             if (limit == limitedOutput) return 0;  /* Check output limit */
+             /* adapt lastRunSize to fill 'dst' */
+             lastRunSize  = (size_t)(oend - op) - 1;
+             litLength = (lastRunSize + 255 - RUN_MASK) / 255;
+             lastRunSize -= litLength;
+         }
+         ip = anchor + lastRunSize;
+
+         if (lastRunSize >= RUN_MASK) {
+             size_t accumulator = lastRunSize - RUN_MASK;
+             *op++ = (RUN_MASK << ML_BITS);
+             for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;
+             *op++ = (BYTE) accumulator;
+         } else {
+             *op++ = (BYTE)(lastRunSize << ML_BITS);
+         }
+         memcpy(op, anchor, lastRunSize);
+         op += lastRunSize;
+     }
+
+     /* End */
+     *srcSizePtr = (int) (((const char*)ip) - source);
+     return (int) ((char*)op-dst);
+
+ _dest_overflow:
+     if (limit == limitedDestSize) {
+         op = opSaved;  /* restore correct out pointer */
+         goto _last_literals;
+     }
+     return 0;
+ }
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
deleted file mode 100644
index 9bd4822..0000000
--- a/lib/lz4opt.h
+++ /dev/null
@@ -1,356 +0,0 @@
-/*
-    lz4opt.h - Optimal Mode of LZ4
-    Copyright (C) 2015-2017, Przemyslaw Skibinski <inikep@gmail.com>
-    Note : this file is intended to be included within lz4hc.c
-
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-       - LZ4 source repository : https://github.com/lz4/lz4
-       - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-
-#define LZ4_OPT_NUM   (1<<12)
-
-typedef struct {
-    int price;
-    int off;
-    int mlen;
-    int litlen;
-} LZ4HC_optimal_t;
-
-
-/* price in bytes */
-LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen)
-{
-    int price = litlen;
-    if (litlen >= (int)RUN_MASK)
-        price += 1 + (litlen-RUN_MASK)/255;
-    return price;
-}
-
-
-/* requires mlen >= MINMATCH */
-LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
-{
-    int price = 1 + 2 ; /* token + 16-bit offset */
-
-    price += LZ4HC_literalsPrice(litlen);
-
-    if (mlen >= (int)(ML_MASK+MINMATCH))
-        price += 1 + (mlen-(ML_MASK+MINMATCH))/255;
-
-    return price;
-}
-
-
-/*-*************************************
-*  Match finder
-***************************************/
-typedef struct {
-    int off;
-    int len;
-} LZ4HC_match_t;
-
-LZ4_FORCE_INLINE
-LZ4HC_match_t LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
-                        const BYTE* ip, const BYTE* const iHighLimit,
-                        int minLen, int nbSearches)
-{
-    LZ4HC_match_t match = { 0 , 0 };
-    const BYTE* matchPtr = NULL;
-    /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
-     * but this won't be the case here, as we define iLowLimit==ip,
-     * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
-                                ip, ip, iHighLimit, minLen, &matchPtr, &ip,
-                                nbSearches, 1 /* patternAnalysis */);
-    if (matchLength <= minLen) return match;
-    match.len = matchLength;
-    match.off = (int)(ip-matchPtr);
-    return match;
-}
-
-
-static int LZ4HC_compress_optimal (
-    LZ4HC_CCtx_internal* ctx,
-    const char* const source,
-    char* dst,
-    int* srcSizePtr,
-    int dstCapacity,
-    int const nbSearches,
-    size_t sufficient_len,
-    limitedOutput_directive limit,
-    int const fullUpdate
-    )
-{
-#define TRAILING_LITERALS 3
-    LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS];   /* this uses a bit too much stack memory to my taste ... */
-
-    const BYTE* ip = (const BYTE*) source;
-    const BYTE* anchor = ip;
-    const BYTE* const iend = ip + *srcSizePtr;
-    const BYTE* const mflimit = iend - MFLIMIT;
-    const BYTE* const matchlimit = iend - LASTLITERALS;
-    BYTE* op = (BYTE*) dst;
-    BYTE* opSaved = (BYTE*) dst;
-    BYTE* oend = op + dstCapacity;
-
-    /* init */
-    DEBUGLOG(5, "LZ4HC_compress_optimal");
-    *srcSizePtr = 0;
-    if (limit == limitedDestSize) oend -= LASTLITERALS;   /* Hack for support LZ4 format restriction */
-    if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
-
-    /* Main Loop */
-    assert(ip - anchor < LZ4_MAX_INPUT_SIZE);
-    while (ip <= mflimit) {
-        int const llen = (int)(ip - anchor);
-        int best_mlen, best_off;
-        int cur, last_match_pos = 0;
-
-        LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
-        if (firstMatch.len==0) { ip++; continue; }
-
-        if ((size_t)firstMatch.len > sufficient_len) {
-            /* good enough solution : immediate encoding */
-            int const firstML = firstMatch.len;
-            const BYTE* const matchPos = ip - firstMatch.off;
-            opSaved = op;
-            if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) )   /* updates ip, op and anchor */
-                goto _dest_overflow;
-            continue;
-        }
-
-        /* set prices for first positions (literals) */
-        {   int rPos;
-            for (rPos = 0 ; rPos < MINMATCH ; rPos++) {
-                int const cost = LZ4HC_literalsPrice(llen + rPos);
-                opt[rPos].mlen = 1;
-                opt[rPos].off = 0;
-                opt[rPos].litlen = llen + rPos;
-                opt[rPos].price = cost;
-                DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
-                            rPos, cost, opt[rPos].litlen);
-        }   }
-        /* set prices using initial match */
-        {   int mlen = MINMATCH;
-            int const matchML = firstMatch.len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
-            int const offset = firstMatch.off;
-            assert(matchML < LZ4_OPT_NUM);
-            for ( ; mlen <= matchML ; mlen++) {
-                int const cost = LZ4HC_sequencePrice(llen, mlen);
-                opt[mlen].mlen = mlen;
-                opt[mlen].off = offset;
-                opt[mlen].litlen = llen;
-                opt[mlen].price = cost;
-                DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup",
-                            mlen, cost, mlen);
-        }   }
-        last_match_pos = firstMatch.len;
-        {   int addLit;
-            for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
-                opt[last_match_pos+addLit].mlen = 1; /* literal */
-                opt[last_match_pos+addLit].off = 0;
-                opt[last_match_pos+addLit].litlen = addLit;
-                opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
-                DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
-                            last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
-        }   }
-
-        /* check further positions */
-        for (cur = 1; cur < last_match_pos; cur++) {
-            const BYTE* const curPtr = ip + cur;
-            LZ4HC_match_t newMatch;
-
-            if (curPtr > mflimit) break;
-            DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
-                    cur, opt[cur].price, opt[cur+1].price, cur+1);
-            if (fullUpdate) {
-                /* not useful to search here if next position has same (or lower) cost */
-                if ( (opt[cur+1].price <= opt[cur].price)
-                  /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */
-                  && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) )
-                    continue;
-            } else {
-                /* not useful to search here if next position has same (or lower) cost */
-                if (opt[cur+1].price <= opt[cur].price) continue;
-            }
-
-            DEBUGLOG(7, "search at rPos:%u", cur);
-            if (fullUpdate)
-                newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
-            else
-                /* only test matches of minimum length; slightly faster, but misses a few bytes */
-                newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
-            if (!newMatch.len) continue;
-
-            if ( ((size_t)newMatch.len > sufficient_len)
-              || (newMatch.len + cur >= LZ4_OPT_NUM) ) {
-                /* immediate encoding */
-                best_mlen = newMatch.len;
-                best_off = newMatch.off;
-                last_match_pos = cur + 1;
-                goto encode;
-            }
-
-            /* before match : set price with literals at beginning */
-            {   int const baseLitlen = opt[cur].litlen;
-                int litlen;
-                for (litlen = 1; litlen < MINMATCH; litlen++) {
-                    int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen);
-                    int const pos = cur + litlen;
-                    if (price < opt[pos].price) {
-                        opt[pos].mlen = 1; /* literal */
-                        opt[pos].off = 0;
-                        opt[pos].litlen = baseLitlen+litlen;
-                        opt[pos].price = price;
-                        DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)",
-                                    pos, price, opt[pos].litlen);
-            }   }   }
-
-            /* set prices using match at position = cur */
-            {   int const matchML = newMatch.len;
-                int ml = MINMATCH;
-
-                assert(cur + newMatch.len < LZ4_OPT_NUM);
-                for ( ; ml <= matchML ; ml++) {
-                    int const pos = cur + ml;
-                    int const offset = newMatch.off;
-                    int price;
-                    int ll;
-                    DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)",
-                                pos, last_match_pos);
-                    if (opt[cur].mlen == 1) {
-                        ll = opt[cur].litlen;
-                        price = ((cur > ll) ? opt[cur - ll].price : 0)
-                              + LZ4HC_sequencePrice(ll, ml);
-                    } else {
-                        ll = 0;
-                        price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
-                    }
-
-                    if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) {
-                        DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
-                                    pos, price, ml);
-                        assert(pos < LZ4_OPT_NUM);
-                        if ( (ml == matchML)  /* last pos of last match */
-                          && (last_match_pos < pos) )
-                            last_match_pos = pos;
-                        opt[pos].mlen = ml;
-                        opt[pos].off = offset;
-                        opt[pos].litlen = ll;
-                        opt[pos].price = price;
-            }   }   }
-            /* complete following positions with literals */
-            {   int addLit;
-                for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
-                    opt[last_match_pos+addLit].mlen = 1; /* literal */
-                    opt[last_match_pos+addLit].off = 0;
-                    opt[last_match_pos+addLit].litlen = addLit;
-                    opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
-                    DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
-            }   }
-        }  /* for (cur = 1; cur <= last_match_pos; cur++) */
-
-        best_mlen = opt[last_match_pos].mlen;
-        best_off = opt[last_match_pos].off;
-        cur = last_match_pos - best_mlen;
-
-encode: /* cur, last_match_pos, best_mlen, best_off must be set */
-        assert(cur < LZ4_OPT_NUM);
-        assert(last_match_pos >= 1);  /* == 1 when only one candidate */
-        DEBUGLOG(6, "reverse traversal, looking for shortest path")
-        DEBUGLOG(6, "last_match_pos = %i", last_match_pos);
-        {   int candidate_pos = cur;
-            int selected_matchLength = best_mlen;
-            int selected_offset = best_off;
-            while (1) {  /* from end to beginning */
-                int const next_matchLength = opt[candidate_pos].mlen;  /* can be 1, means literal */
-                int const next_offset = opt[candidate_pos].off;
-                DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
-                opt[candidate_pos].mlen = selected_matchLength;
-                opt[candidate_pos].off = selected_offset;
-                selected_matchLength = next_matchLength;
-                selected_offset = next_offset;
-                if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */
-                assert(next_matchLength > 0);  /* can be 1, means literal */
-                candidate_pos -= next_matchLength;
-        }   }
-
-        /* encode all recorded sequences in order */
-        {   int rPos = 0;  /* relative position (to ip) */
-            while (rPos < last_match_pos) {
-                int const ml = opt[rPos].mlen;
-                int const offset = opt[rPos].off;
-                if (ml == 1) { ip++; rPos++; continue; }  /* literal; note: can end up with several literals, in which case, skip them */
-                rPos += ml;
-                assert(ml >= MINMATCH);
-                assert((offset >= 1) && (offset <= MAX_DISTANCE));
-                opSaved = op;
-                if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) )   /* updates ip, op and anchor */
-                    goto _dest_overflow;
-        }   }
-    }  /* while (ip <= mflimit) */
-
-_last_literals:
-    /* Encode Last Literals */
-    {   size_t lastRunSize = (size_t)(iend - anchor);  /* literals */
-        size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
-        size_t const totalSize = 1 + litLength + lastRunSize;
-        if (limit == limitedDestSize) oend += LASTLITERALS;  /* restore correct value */
-        if (limit && (op + totalSize > oend)) {
-            if (limit == limitedOutput) return 0;  /* Check output limit */
-            /* adapt lastRunSize to fill 'dst' */
-            lastRunSize  = (size_t)(oend - op) - 1;
-            litLength = (lastRunSize + 255 - RUN_MASK) / 255;
-            lastRunSize -= litLength;
-        }
-        ip = anchor + lastRunSize;
-
-        if (lastRunSize >= RUN_MASK) {
-            size_t accumulator = lastRunSize - RUN_MASK;
-            *op++ = (RUN_MASK << ML_BITS);
-            for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;
-            *op++ = (BYTE) accumulator;
-        } else {
-            *op++ = (BYTE)(lastRunSize << ML_BITS);
-        }
-        memcpy(op, anchor, lastRunSize);
-        op += lastRunSize;
-    }
-
-    /* End */
-    *srcSizePtr = (int) (((const char*)ip) - source);
-    return (int) ((char*)op-dst);
-
-_dest_overflow:
-    if (limit == limitedDestSize) {
-        op = opSaved;  /* restore correct out pointer */
-        goto _last_literals;
-    }
-    return 0;
-}
-- 
cgit v0.12


From ba115386fa69c148cd1b27790a8b95cd817ff172 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 26 Feb 2018 13:31:18 -0800
Subject: update code comment on LZ4 streaming interface

notably regarding LZ4_saveDict() speed advantage,
answering #477.
---
 doc/lz4_manual.html | 21 +++++++++++----------
 lib/lz4.h           | 23 ++++++++++++-----------
 2 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index b5a4042..43107c6 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -172,28 +172,29 @@ int           LZ4_freeStream (LZ4_stream_t* streamPtr);
 </p></pre><BR>
 
 <pre><b>int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
-</b><p>  Compress content into 'src' using data from previously compressed blocks, improving compression ratio.
+</b><p>  Compress 'src' content using data from previously compressed blocks, for better compression ratio.
   'dst' buffer must be already allocated.
   If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
 
-  Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
-              If less than 64KB has been compressed all the data must be present.
+  Important : The previous 64KB of compressed data is assumed to remain present and unmodified in memory!
+
   Special 1 : When input is a double-buffer, they can have any size, including < 64 KB.
               Make sure that buffers are separated by at least one byte.
-              This way, rule becomes simple : each block depends on previous block only.
+              This way, each block only depends on previous block.
   Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
 
  @return : size of compressed block
-           or 0 if there is an error (typically, compressed data cannot fit into 'dst')
+           or 0 if there is an error (typically, cannot fit into 'dst').
   After an error, the stream status is invalid, it can only be reset or freed.
  
 </p></pre><BR>
 
-<pre><b>int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
-</b><p>  If previously compressed data block is not guaranteed to remain available at its current memory location,
+<pre><b>int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize);
+</b><p>  If last 64KB data cannot be guaranteed to remain available at its current memory location,
   save it into a safer place (char* safeBuffer).
-  Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable.
-  @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+  This is schematically equivalent to a memcpy() followed by LZ4_loadDict(),
+  but is much faster, because LZ4_saveDict() doesn't need to rebuild tables.
+ @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error.
  
 </p></pre><BR>
 
@@ -209,7 +210,7 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
 <pre><b>int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
 </b><p>  An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
   Use this function to start decompression of a new stream of blocks.
-  A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order.
+  A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
  @return : 1 if OK, 0 if error
  
 </p></pre><BR>
diff --git a/lib/lz4.h b/lib/lz4.h
index 08f06c7..2dd1ff5 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -236,7 +236,7 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcS
 /*-*********************************************
 *  Streaming Compression Functions
 ***********************************************/
-typedef union LZ4_stream_u LZ4_stream_t;   /* incomplete type (defined later) */
+typedef union LZ4_stream_u LZ4_stream_t;  /* incomplete type (defined later) */
 
 /*! LZ4_createStream() and LZ4_freeStream() :
  *  LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure.
@@ -260,30 +260,31 @@ LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
 LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
 
 /*! LZ4_compress_fast_continue() :
- *  Compress content into 'src' using data from previously compressed blocks, improving compression ratio.
+ *  Compress 'src' content using data from previously compressed blocks, for better compression ratio.
  *  'dst' buffer must be already allocated.
  *  If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
  *
- *  Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
- *              If less than 64KB has been compressed all the data must be present.
+ *  Important : The previous 64KB of compressed data is assumed to remain present and unmodified in memory!
+ *
  *  Special 1 : When input is a double-buffer, they can have any size, including < 64 KB.
  *              Make sure that buffers are separated by at least one byte.
- *              This way, rule becomes simple : each block depends on previous block only.
+ *              This way, each block only depends on previous block.
  *  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
  *
  * @return : size of compressed block
- *           or 0 if there is an error (typically, compressed data cannot fit into 'dst')
+ *           or 0 if there is an error (typically, cannot fit into 'dst').
  *  After an error, the stream status is invalid, it can only be reset or freed.
  */
 LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 /*! LZ4_saveDict() :
- *  If previously compressed data block is not guaranteed to remain available at its current memory location,
+ *  If last 64KB data cannot be guaranteed to remain available at its current memory location,
  *  save it into a safer place (char* safeBuffer).
- *  Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable.
- *  @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+ *  This is schematically equivalent to a memcpy() followed by LZ4_loadDict(),
+ *  but is much faster, because LZ4_saveDict() doesn't need to rebuild tables.
+ * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error.
  */
-LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
+LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize);
 
 
 /*-**********************************************
@@ -301,7 +302,7 @@ LZ4LIB_API int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str
 /*! LZ4_setStreamDecode() :
  *  An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
  *  Use this function to start decompression of a new stream of blocks.
- *  A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order.
+ *  A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
  * @return : 1 if OK, 0 if error
  */
 LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
-- 
cgit v0.12


From 39fda9a447d4dd51f395c5d94f9b63c7aec9b68b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 26 Feb 2018 13:50:04 -0800
Subject: bumped version number to v1.8.2

updated NEWS was current progresses
---
 NEWS                     | 5 +++++
 doc/lz4_manual.html      | 4 ++--
 doc/lz4frame_manual.html | 4 ++--
 lib/lz4.h                | 2 +-
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/NEWS b/NEWS
index f9c503f..9ac41f2 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,8 @@
+v1.8.2
+perf: slightly faster HC compression and decompression speed
+perf: very small compression ratio improvement
+cli : benchmark mode more accurate for small inputs
+
 v1.8.1
 perf : faster and stronger ultra modes (levels 10+)
 perf : slightly faster compression and decompression speed
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 43107c6..d14467f 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>1.8.1 Manual</title>
+<title>1.8.2 Manual</title>
 </head>
 <body>
-<h1>1.8.1 Manual</h1>
+<h1>1.8.2 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 1ddf6d8..db004ab 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>1.8.1 Manual</title>
+<title>1.8.2 Manual</title>
 </head>
 <body>
-<h1>1.8.1 Manual</h1>
+<h1>1.8.2 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
diff --git a/lib/lz4.h b/lib/lz4.h
index 2dd1ff5..911ea79 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -93,7 +93,7 @@ extern "C" {
 /*------   Version   ------*/
 #define LZ4_VERSION_MAJOR    1    /* for breaking interface changes  */
 #define LZ4_VERSION_MINOR    8    /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE  1    /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_RELEASE  2    /* for tweaks, bug-fixes, or development */
 
 #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
 
-- 
cgit v0.12


From 0ddd1ceb1db0f3a40a5231d52f59d25ceacd6480 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 26 Feb 2018 14:09:46 -0800
Subject: added target make check

according to GNU Makefile conventions,
the Makefile should feature a make check target
to self-test the generated program:
https://www.gnu.org/prep/standards/html_node/Standard-Targets.html .

this is much less thorough and less taxing than `make test`,
and can be run on any target in a reasonable timeframe (several seconds).
---
 Makefile       |  8 +++++---
 tests/Makefile | 29 ++++++++++++++++-------------
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/Makefile b/Makefile
index 83320fa..8a7d155 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,8 @@
 # ################################################################
 # LZ4 - Makefile
-# Copyright (C) Yann Collet 2011-2016
+# Copyright (C) Yann Collet 2011-present
 # All rights reserved.
 #
-# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
-#
 # BSD license
 # Redistribution and use in source and binary forms, with or without modification,
 # are permitted provided that the following conditions are met:
@@ -122,6 +120,10 @@ ifneq (,$(filter $(HOST_OS),MSYS POSIX))
 list:
 	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
 
+.PHONY: check
+check:
+	$(MAKE) -C $(TESTDIR) test-lz4-essentials
+
 .PHONY: test
 test:
 	$(MAKE) -C $(TESTDIR) $@
diff --git a/tests/Makefile b/tests/Makefile
index ddc0d2e..5954370 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,8 +1,6 @@
 # ##########################################################################
 # LZ4 programs - Makefile
-# Copyright (C) Yann Collet 2011-2017
-#
-# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
+# Copyright (C) Yann Collet 2011-present
 #
 # GPL v2 License
 #
@@ -53,7 +51,7 @@ else
 EXT  =
 VOID = /dev/null
 endif
-LZ4     := $(PRGDIR)/lz4$(EXT)
+LZ4 := $(PRGDIR)/lz4$(EXT)
 
 
 # Default test parameters
@@ -184,12 +182,6 @@ test-lz4-contentSize: lz4 datagen
 	$(LZ4) -v tmplc1 | $(LZ4) -t
 	$(LZ4) -v --content-size tmplc1 | $(LZ4) -d > tmplc2
 	$(DIFF) -s tmplc1 tmplc2
-	# test large size [2-4] GB
-	@./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmplc1
-	@ls -ls tmplc1
-	@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmplc2
-	@ls -ls tmplc2
-	$(DIFF) -s tmplc1 tmplc2
 	@$(RM) tmplc*
 
 test-lz4-frame-concatenation: lz4 datagen
@@ -293,6 +285,13 @@ test-lz4-hugefile: lz4 datagen
 	@echo "\n ---- test huge files compression/decompression ----"
 	./datagen -g6GB   | $(LZ4) -vB5D  | $(LZ4) -qt
 	./datagen -g6GB   | $(LZ4) -v5BD  | $(LZ4) -qt
+	# test large file size [2-4] GB
+	@./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1
+	@ls -ls tmphf1
+	@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmphf2
+	@ls -ls tmphf2
+	$(DIFF) -s tmphf1 tmphf2
+	@$(RM) tmphf*
 
 test-lz4-testmode: lz4 datagen
 	@echo "\n ---- bench mode ----"
@@ -326,9 +325,13 @@ test-lz4-opt-parser: lz4 datagen
 	./datagen -g16M -P90  | $(LZ4) -11B5    | $(LZ4) -t
 	./datagen -g32M -P10  | $(LZ4) -11B5D   | $(LZ4) -t
 
-test-lz4: lz4 datagen test-lz4-basic test-lz4-opt-parser test-lz4-multiple \
-          test-lz4-sparse test-lz4-frame-concatenation test-lz4-testmode \
-          test-lz4-contentSize test-lz4-hugefile test-lz4-dict
+test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple \
+                      test-lz4-frame-concatenation test-lz4-testmode \
+                      test-lz4-contentSize
+	@$(RM) tmp*
+
+test-lz4: lz4 datagen test-lz4-essentials test-lz4-opt-parser \
+          test-lz4-sparse test-lz4-hugefile test-lz4-dict
 	@$(RM) tmp*
 
 test-lz4c: lz4c datagen
-- 
cgit v0.12


From b5233d3726b416b1176c71483d20b4c543851c6f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 27 Feb 2018 23:23:27 -0800
Subject: updated LZ4F_compressBound() documentation

to clarify it includes potentially buffered data.
---
 doc/lz4frame_manual.html | 9 ++++++---
 lib/lz4frame.h           | 9 ++++++---
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index db004ab..f0f7f5a 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -153,10 +153,13 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 </p></pre><BR>
 
 <pre><b>size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
-</b><p>  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
-  Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
-  Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
+</b><p>  Provides minimum dstCapacity required to guarantee compression success
+  given a srcSize and preferences, covering worst case scenario.
   prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
+  Estimation is valid for either LZ4F_compressUpdate(), LZ4F_flush() or LZ4F_compressEnd(),
+  Estimation includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes.
+  It also includes frame footer (ending + checksum), which would have to be generated by LZ4F_compressEnd().
+  Estimation doesn't include frame header, as it was already generated by LZ4F_compressBegin().
   Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
   When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
  
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 15484d7..9efaca0 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -250,10 +250,13 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
                                       const LZ4F_preferences_t* prefsPtr);
 
 /*! LZ4F_compressBound() :
- *  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
- *  Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
- *  Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
+ *  Provides minimum dstCapacity required to guarantee compression success
+ *  given a srcSize and preferences, covering worst case scenario.
  *  prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
+ *  Estimation is valid for either LZ4F_compressUpdate(), LZ4F_flush() or LZ4F_compressEnd(),
+ *  Estimation includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes.
+ *  It also includes frame footer (ending + checksum), which would have to be generated by LZ4F_compressEnd().
+ *  Estimation doesn't include frame header, as it was already generated by LZ4F_compressBegin().
  *  Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
  *  When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
  */
-- 
cgit v0.12


From 6d4e60e365d1353c9dac46f5bcfa48edfd9ab623 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 9 Mar 2018 09:57:29 -0800
Subject: fix #481: ensure liblz4.a dependency for `make all`

`make all` will trigger several sub-directory makefiles.
several of them need `liblz4.a`.
When built with `-j#`, there are several concurrent liblz4.a built

Make liblz4.a a dependency, which is built once,
before moving to sub-directory Makefiles
---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 8a7d155..ff84d6d 100644
--- a/Makefile
+++ b/Makefile
@@ -56,6 +56,7 @@ all: allmost manuals
 allmost: lib lz4 examples
 
 .PHONY: lib lib-release liblz4.a
+lib: liblz4.a
 lib lib-release liblz4.a:
 	@$(MAKE) -C $(LZ4DIR) $@
 
@@ -67,7 +68,7 @@ lz4 lz4-release :
 	@cp $(PRGDIR)/lz4$(EXT) .
 
 .PHONY: examples
-examples:
+examples: liblz4.a
 	$(MAKE) -C $(EXDIR) all
 
 .PHONY: manuals
-- 
cgit v0.12


From 6c23f03b93608a444f8747940bca26708a58fa78 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 9 Mar 2018 11:54:32 -0800
Subject: fix #482: change CFLAGS to CXXFLAGS

as they are associated with $(CXX)
---
 contrib/gen_manual/Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/contrib/gen_manual/Makefile b/contrib/gen_manual/Makefile
index 9fbe858..95abe2e 100644
--- a/contrib/gen_manual/Makefile
+++ b/contrib/gen_manual/Makefile
@@ -30,10 +30,10 @@
 # ################################################################
 
 
-CFLAGS ?= -O3
-CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
-CFLAGS += $(MOREFLAGS)
-FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+CXXFLAGS ?= -O3
+CXXFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
+CXXFLAGS += $(MOREFLAGS)
+FLAGS   = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
 
 LZ4API = ../../lib/lz4.h
 LZ4MANUAL = ../../doc/lz4_manual.html
-- 
cgit v0.12


From 8c006b19bbed0b3bb2466dc8af2a603ca0ba7fab Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 29 Jan 2018 13:20:16 -0500
Subject: Add a Benchmarking Tool For Compression with Context Re-Use

---
 tests/Makefile     |   8 +-
 tests/framebench.c | 293 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 299 insertions(+), 2 deletions(-)
 create mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index 5954370..77d9e36 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -62,7 +62,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen
+all: fullbench fuzzer frametest datagen framebench
 
 all32: CFLAGS+=-m32
 all32: all
@@ -98,6 +98,9 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
+framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -109,7 +112,8 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
-        fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
+        framebench$(EXT) \
+        fasttest$(EXT) datagen$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
 
diff --git a/tests/framebench.c b/tests/framebench.c
new file mode 100644
index 0000000..8dcfa41
--- /dev/null
+++ b/tests/framebench.c
@@ -0,0 +1,293 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "lz4.h"
+#include "lz4frame.h"
+#include "lz4frame_static.h"
+
+#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
+
+typedef struct {
+  size_t iter;
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  char *obuf;
+  size_t osize;
+  const char* ibuf;
+  size_t isize;
+  size_t num_ibuf;
+  const LZ4F_CDict* cdict;
+  LZ4F_preferences_t* prefs;
+  const LZ4F_compressOptions_t* options;
+} bench_params_t;
+
+size_t compress_frame(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressFrame_usingCDict(
+    obuf,
+    osize,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    cdict,
+    prefs);
+  LZ4F_CHECK(oused);
+
+  return oused;
+}
+
+size_t compress_begin(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+  const LZ4F_compressOptions_t* options = p->options;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressUpdate(
+    cctx,
+    obuf,
+    oend - obuf,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    options);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
+  LZ4F_CHECK(oused);
+
+  return obuf - p->obuf;
+}
+
+size_t compress_default(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_extState(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4_stream_t *ctx = p->ctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_fast_extState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+uint64_t bench(
+    size_t (*fun)(bench_params_t *),
+    uint64_t repetitions,
+    bench_params_t *params,
+    size_t *osizePtr
+) {
+  struct timespec start, end;
+  size_t i, osize = 0;
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
+
+  for (i = 0; i < repetitions; i++) {
+    size_t o;
+    params->iter = i;
+    o = fun(params);
+    if (!o) return 0;
+    osize += o;
+  }
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+
+  *osizePtr = osize / repetitions;
+  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+}
+
+int main(int argc, char *argv[]) {
+
+
+  struct stat st;
+  size_t bytes_read;
+
+  const char *dict_fn;
+  size_t dict_size;
+  char *dict_buf;
+  FILE *dict_file;
+
+  const char *in_fn;
+  size_t in_size;
+  size_t num_in_buf;
+  size_t cur_in_buf;
+  char *in_buf;
+  FILE *in_file;
+
+  size_t out_size;
+  char *out_buf;
+
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  LZ4F_CDict *cdict;
+  LZ4F_preferences_t prefs;
+  LZ4F_compressOptions_t options;
+
+  size_t out_used;
+
+  uint64_t time_taken;
+  uint64_t repetitions;
+
+  bench_params_t params;
+
+  if (argc != 3) return 1;
+  dict_fn = argv[1];
+  in_fn = argv[2];
+
+  if (stat(dict_fn, &st)) return 1;
+  dict_size = st.st_size;
+  dict_buf = (char *)malloc(dict_size);
+  if (!dict_buf) return 1;
+  dict_file = fopen(dict_fn, "r");
+  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
+  if (bytes_read != dict_size) return 1;
+
+  if (stat(in_fn, &st)) return 1;
+  in_size = st.st_size;
+  num_in_buf = 256 * 1024 * 1024 / in_size;
+  if (num_in_buf == 0) {
+    num_in_buf = 1;
+  }
+
+  in_buf = (char *)malloc(in_size * num_in_buf);
+  if (!in_buf) return 1;
+  in_file = fopen(in_fn, "r");
+  bytes_read = fread(in_buf, 1, in_size, in_file);
+  if (bytes_read != in_size) return 1;
+
+  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
+    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
+  }
+
+  if (in_size <= 1024) {
+    repetitions = 100000;
+  } else
+  if (in_size <= 16384) {
+    repetitions = 10000;
+  } else
+  if (in_size <= 131072) {
+    repetitions = 1000;
+  } else
+  if (in_size <= 1048576) {
+    repetitions = 100;
+  } else {
+    repetitions = 50;
+  }
+
+  memset(&prefs, 0, sizeof(prefs));
+  prefs.autoFlush = 1;
+  if (in_size < 64 * 1024)
+    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
+  prefs.frameInfo.contentSize = in_size;
+
+  memset(&options, 0, sizeof(options));
+  options.stableSrc = 1;
+
+  out_size = LZ4F_compressFrameBound(in_size, &prefs);
+  out_buf = (char *)malloc(out_size);
+  if (!out_buf) return 1;
+
+  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
+  if (cctx == NULL) return 1;
+
+  ctx = LZ4_createStream();
+  if (ctx == NULL) return 1;
+
+  cdict = LZ4F_createCDict(dict_buf, dict_size);
+  if (!cdict) return 1;
+
+  fprintf(stderr, "dict  size: %zd\n", dict_size);
+  fprintf(stderr, "input size: %zd\n", in_size);
+
+  params.ctx = ctx;
+  params.cctx = cctx;
+  params.obuf = out_buf;
+  params.osize = out_size;
+  params.ibuf = in_buf;
+  params.isize = in_size;
+  params.num_ibuf = num_in_buf;
+  params.cdict = NULL;
+  params.prefs = &prefs;
+  params.options = &options;
+
+  time_taken = bench(compress_default, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_extState, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  params.cdict = cdict;
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  return 0;
+}
\ No newline at end of file
-- 
cgit v0.12


From 6d156fea56debbde1699b54a52c4907ff0b96017 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Oct 2017 16:13:33 -0400
Subject: Allow Empty Dictionaries

---
 lib/lz4.c | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 38a865f..898b82a 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1018,12 +1018,6 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     if ((dict->initCheck) || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
         LZ4_resetStream(LZ4_dict);
 
-    if (dictSize < (int)HASH_UNIT) {
-        dict->dictionary = NULL;
-        dict->dictSize = 0;
-        return 0;
-    }
-
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
     dict->currentOffset += 64 KB;
     base = p - dict->currentOffset;
@@ -1031,6 +1025,10 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     dict->dictSize = (U32)(dictEnd - p);
     dict->currentOffset += dict->dictSize;
 
+    if (dictSize < (int)HASH_UNIT) {
+        return 0;
+    }
+
     while (p <= dictEnd-HASH_UNIT) {
         LZ4_putPosition(p, dict->hashTable, byU32, base);
         p+=3;
-- 
cgit v0.12


From 6933f5ad9ca57caa8a76103f1382560fba0f9691 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 17 Oct 2017 15:23:02 -0400
Subject: Remove Obsolete Stream Functions to Free Space in LZ4_stream_t

---
 lib/lz4.c | 31 -------------------------------
 lib/lz4.h |  6 ------
 2 files changed, 37 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 898b82a..6168a2a 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1515,37 +1515,6 @@ int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4
 int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
 
 
-/* Obsolete Streaming functions */
-
-int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
-
-static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base)
-{
-    MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t));
-    lz4ds->internal_donotuse.bufferStart = base;
-}
-
-int LZ4_resetStreamState(void* state, char* inputBuffer)
-{
-    if ((((uptrval)state) & 3) != 0) return 1;   /* Error : pointer is not aligned on 4-bytes boundary */
-    LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer);
-    return 0;
-}
-
-void* LZ4_create (char* inputBuffer)
-{
-    LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t));
-    LZ4_init (lz4ds, (BYTE*)inputBuffer);
-    return lz4ds;
-}
-
-char* LZ4_slideInputBuffer (void* LZ4_Data)
-{
-    LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse;
-    int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB);
-    return (char*)(ctx->bufferStart + dictSize);
-}
-
 /* Obsolete streaming decompression functions */
 
 int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
diff --git a/lib/lz4.h b/lib/lz4.h
index 911ea79..e5d4039 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -465,12 +465,6 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
 LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
-/* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
-LZ4_DEPRECATED("use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
-
 /* Obsolete streaming decoding functions */
 LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
 LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
-- 
cgit v0.12


From 5709891de6513ee8253f27e8d3207ec1fdcff14b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:11:35 -0500
Subject: Add a Table Type Field to LZ4_stream_t

---
 lib/lz4.c | 24 ++++++++++++++++++------
 lib/lz4.h |  6 ++++--
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 6168a2a..7b8de77 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -446,7 +446,7 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 *  Local Structures and types
 **************************************/
 typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
-typedef enum { byPtr, byU32, byU16 } tableType_t;
+typedef enum { unusedTable = 0, byPtr = 1, byU32 = 2, byU16 = 3 } tableType_t;
 
 typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive;
 typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
@@ -496,6 +496,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy
 {
     switch (tableType)
     {
+    case unusedTable: { /* illegal! */ assert(0); return; }
     case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; }
     case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; }
     case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; }
@@ -551,6 +552,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
 
+    ptrdiff_t retval = 0;
+
     U32 forwardH;
 
     /* Init conditions */
@@ -622,7 +625,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             token = op++;
             if ((outputLimited) &&  /* Check output buffer overflow */
                 (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
-                return 0;
+                goto _clean_up;
             if (litLength >= RUN_MASK) {
                 int len = (int)litLength-RUN_MASK;
                 *token = (RUN_MASK<<ML_BITS);
@@ -662,7 +665,7 @@ _next_match:
 
             if ( outputLimited &&    /* Check output buffer overflow */
                 (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
-                return 0;
+                goto _clean_up;
             if (matchCode >= ML_MASK) {
                 *token += ML_MASK;
                 matchCode -= ML_MASK;
@@ -711,7 +714,7 @@ _last_literals:
     {   size_t const lastRun = (size_t)(iend - anchor);
         if ( (outputLimited) &&  /* Check output buffer overflow */
             ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) )
-            return 0;
+            goto _clean_up;
         if (lastRun >= RUN_MASK) {
             size_t accumulator = lastRun - RUN_MASK;
             *op++ = RUN_MASK << ML_BITS;
@@ -724,8 +727,13 @@ _last_literals:
         op += lastRun;
     }
 
+    retval = (((char*)op)-dest);
+
+_clean_up:
+    cctx->tableType = tableType;
+
     /* End */
-    return (int) (((char*)op)-dest);
+    return (int)retval;
 }
 
 
@@ -997,6 +1005,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(4, "LZ4_resetStream");
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
+    LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
@@ -1015,7 +1024,9 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
-    if ((dict->initCheck) || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
+    if ((dict->initCheck)
+      || (dict->tableType != byU32 && dict->tableType != unusedTable)
+      || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
         LZ4_resetStream(LZ4_dict);
 
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
@@ -1024,6 +1035,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     dict->dictionary = p;
     dict->dictSize = (U32)(dictEnd - p);
     dict->currentOffset += dict->dictSize;
+    dict->tableType = byU32;
 
     if (dictSize < (int)HASH_UNIT) {
         return 0;
diff --git a/lib/lz4.h b/lib/lz4.h
index e5d4039..d2e2103 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -360,7 +360,8 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 typedef struct {
     uint32_t hashTable[LZ4_HASH_SIZE_U32];
     uint32_t currentOffset;
-    uint32_t initCheck;
+    uint16_t initCheck;
+    uint16_t tableType;
     const uint8_t* dictionary;
     uint8_t* bufferStart;   /* obsolete, used for slideInputBuffer */
     uint32_t dictSize;
@@ -378,7 +379,8 @@ typedef struct {
 typedef struct {
     unsigned int hashTable[LZ4_HASH_SIZE_U32];
     unsigned int currentOffset;
-    unsigned int initCheck;
+    unsigned short initCheck;
+    unsigned short tableType;
     const unsigned char* dictionary;
     unsigned char* bufferStart;   /* obsolete, used for slideInputBuffer */
     unsigned int dictSize;
-- 
cgit v0.12


From f34fb3c42dbb824af97800eefdb487dff31e0c13 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:25:29 -0500
Subject: Add Bounds Check to locateBuffDiff

---
 tests/frametest.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/frametest.c b/tests/frametest.c
index 88d0afd..986a16d 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -730,15 +730,17 @@ _output_error:
 
 static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
 {
-    int p=0;
+    size_t p=0;
     const BYTE* b1=(const BYTE*)buff1;
     const BYTE* b2=(const BYTE*)buff2;
     if (nonContiguous) {
         DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size);
         return;
     }
-    while (b1[p]==b2[p]) p++;
-    DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]);
+    while (p < size && b1[p]==b2[p]) p++;
+    if (p != size) {
+        DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]);
+    }
 }
 
 
-- 
cgit v0.12


From d6a3024dbb6375190e6e02ad8b06aae28e76b946 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:41:11 -0500
Subject: Add LZ4_compress_fast_safeExtState Function

---
 lib/lz4.c | 10 +++++++++-
 lib/lz4.h |  8 ++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 7b8de77..45cc7c9 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -739,9 +739,17 @@ _clean_up:
 
 int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
+  LZ4_resetStream((LZ4_stream_t*)state);
+  return LZ4_compress_fast_safeExtState(state, source, dest, inputSize, maxOutputSize, acceleration);
+}
+
+
+int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+{
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
-    LZ4_resetStream((LZ4_stream_t*)state);
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
+    ctx->dictionary = NULL;
+    ctx->dictSize = 0;
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit)
diff --git a/lib/lz4.h b/lib/lz4.h
index d2e2103..0013ffe 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -179,13 +179,21 @@ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int d
 
 
 /*!
+LZ4_compress_fast_safeExtState() :
 LZ4_compress_fast_extState() :
     Same compression function, just using an externally allocated memory space to store compression state.
     Use LZ4_sizeofState() to know how much memory must be allocated,
     and allocate it on 8-bytes boundaries (using malloc() typically).
     Then, provide it as 'void* state' to compression function.
+
+    Use _safeExtState variant if LZ4_resetStream() was called on the state
+    buffer before being used for the first time (calls to this function leave
+    the state in a safe state, so zeroing is not required between calls).
+    Otherwise, using legacy _extState requires LZ4 to reinitialize the state
+    internally for every call.
 */
 LZ4LIB_API int LZ4_sizeofState(void);
+LZ4LIB_API int LZ4_compress_fast_safeExtState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
-- 
cgit v0.12


From aa36e118f10d825ceb3899d3e137d643ed469331 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:45:59 -0500
Subject: Const-ify Table Arg to LZ4_getPosition(OnHash)

---
 lib/lz4.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 45cc7c9..46935f8 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -509,14 +509,14 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_
     LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
 }
 
-static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
-    if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
-    if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
-    { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
+    if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; }
+    if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; return hashTable[h] + srcBase; }
+    { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
 }
 
-LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     U32 const h = LZ4_hashPosition(p, tableType);
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
-- 
cgit v0.12


From b3628cb0c51bb93f681707b2d0cdbce5ccdae818 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:09:38 -0500
Subject: Avoid Resetting the Context When Possible

---
 lib/lz4.c          | 61 ++++++++++++++++++++++++++++++++++++++----------------
 lib/lz4frame.c     | 21 +++++++++++++++++--
 tests/framebench.c |  2 +-
 3 files changed, 63 insertions(+), 21 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 46935f8..d558b3b 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -538,7 +538,20 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                  const U32 acceleration)
 {
     const BYTE* ip = (const BYTE*) source;
-    const BYTE* base;
+
+    /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
+     * therefore safe to use no matter what mode we're in. Otherwise, we figure
+     * out if it's safe to leave as is or whether it needs to be reset.
+     */
+    const int resetTable = cctx->tableType != unusedTable && (
+                           cctx->tableType != tableType ||
+                            (tableType == byU16 &&
+                             cctx->currentOffset + inputSize >= 0xFFFFU) ||
+                           tableType == byPtr);
+
+    size_t currentOffset = ((tableType == byU32 || tableType == byU16) &&
+                            !resetTable) ? cctx->currentOffset : 0;
+    const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
     const BYTE* const lowRefLimit = ip - cctx->dictSize;
     const BYTE* const dictionary = cctx->dictionary;
@@ -556,17 +569,18 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     U32 forwardH;
 
+    /* TODO: resurrect dictIssue check */
+    (void)dictIssue;
+
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     switch(dict)
     {
     case noDict:
     default:
-        base = (const BYTE*)source;
         lowLimit = (const BYTE*)source;
         break;
     case withPrefix64k:
-        base = (const BYTE*)source - cctx->currentOffset;
         lowLimit = (const BYTE*)source - cctx->dictSize;
         break;
     case usingExtDict:
@@ -575,6 +589,13 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         break;
     }
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
+
+    if (resetTable) {
+      DEBUGLOG(4, "Resetting table in %p", cctx);
+      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
+      cctx->currentOffset = 0;
+    }
+
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* First Byte */
@@ -612,7 +633,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
 
-            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
+            } while ( match < lowRefLimit
                 || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
         }
@@ -700,7 +721,7 @@ _next_match:
                 lowLimit = (const BYTE*)source;
         }   }
         LZ4_putPosition(ip, cctx->hashTable, tableType, base);
-        if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
+        if ( match >= lowRefLimit
             && (match+MAX_DISTANCE>=ip)
             && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
         { token=op++; *token=0; goto _next_match; }
@@ -730,6 +751,8 @@ _last_literals:
     retval = (((char*)op)-dest);
 
 _clean_up:
+    cctx->currentOffset += (U32)inputSize;
+    cctx->dictSize += (U32)inputSize;
     cctx->tableType = tableType;
 
     /* End */
@@ -767,14 +790,16 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
 
 int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
+    int result;
 #if (LZ4_HEAPMODE)
-    void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    ctxPtr->internal_donotuse.currentOffset = 0;
 #else
     LZ4_stream_t ctx;
-    void* const ctxPtr = &ctx;
+    LZ4_stream_t* const ctxPtr = &ctx;
 #endif
-
-    int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
+    LZ4_resetStream(ctxPtr);
+    result = LZ4_compress_fast_safeExtState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
 
 #if (LZ4_HEAPMODE)
     FREEMEM(ctxPtr);
@@ -1005,20 +1030,23 @@ LZ4_stream_t* LZ4_createStream(void)
 {
     LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64);
     LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal));    /* A compilation error here means LZ4_STREAMSIZE is not large enough */
+    DEBUGLOG(4, "LZ4_createStream %p", lz4s);
     LZ4_resetStream(lz4s);
     return lz4s;
 }
 
 void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
-    DEBUGLOG(4, "LZ4_resetStream");
+    DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
+    LZ4_stream->internal_donotuse.currentOffset = 0;
     LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
 {
     if (!LZ4_stream) return 0;   /* support free on NULL */
+    DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream);
     FREEMEM(LZ4_stream);
     return (0);
 }
@@ -1032,6 +1060,8 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
+    DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
+
     if ((dict->initCheck)
       || (dict->tableType != byU32 && dict->tableType != unusedTable)
       || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
@@ -1066,6 +1096,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
         U32 const delta = LZ4_dict->currentOffset - 64 KB;
         const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
         int i;
+        DEBUGLOG(4, "LZ4_renormDictT %p", LZ4_dict);
         for (i=0; i<LZ4_HASH_SIZE_U32; i++) {
             if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0;
             else LZ4_dict->hashTable[i] -= delta;
@@ -1100,14 +1131,10 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
-        int result;
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
         else
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
-        streamPtr->dictSize += (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
-        return result;
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
     }
 
     /* external dictionary mode */
@@ -1118,7 +1145,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
             result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
         streamPtr->dictionary = (const BYTE*)source;
         streamPtr->dictSize = (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
         return result;
     }
 }
@@ -1139,7 +1165,6 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
 
     streamPtr->dictionary = (const BYTE*)source;
     streamPtr->dictSize = (U32)inputSize;
-    streamPtr->currentOffset += (U32)inputSize;
 
     return result;
 }
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a394d1f..bf70b5c 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -354,6 +354,7 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
         prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* only one block => no need for inter-block link */
 
     if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+        LZ4_resetStream(&lz4ctx);
         cctxI.lz4CtxPtr = &lz4ctx;
         cctxI.lz4CtxLevel = 1;
     }  /* fast compression context pre-created on stack */
@@ -521,7 +522,14 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
             if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             cctxPtr->lz4CtxLevel = ctxTypeID;
-    }   }
+        } else if (cctxPtr->lz4CtxLevel != ctxTypeID) {
+            /* otherwise, we must be transitioning from HC -> LZ4.
+             * In that case, avoid reallocating, since a LZ4 ctx
+             * fits in an HC ctx. Just reset. */
+            LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
+            cctxPtr->lz4CtxLevel = ctxTypeID;
+        }
+    }
 
     /* Buffer Management */
     if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
@@ -654,11 +662,20 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
+    LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)ctx)->internal_donotuse;
+    assert(!internal_ctx->initCheck);
+    if (internal_ctx->currentOffset > 1 GB) {
+        /* Init the context */
+        LZ4_resetStream((LZ4_stream_t*)ctx);
+    }
+    /* Clear any local dictionary */
+    internal_ctx->dictionary = NULL;
+    internal_ctx->dictSize = 0;
     if (cdict) {
         memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
-    return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration);
+    return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
 static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
diff --git a/tests/framebench.c b/tests/framebench.c
index 8dcfa41..21c3704 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -118,7 +118,7 @@ size_t compress_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_extState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  oused = LZ4_compress_fast_safeExtState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
   obuf += oused;
 
   return obuf - p->obuf;
-- 
cgit v0.12


From 7060bcabf041395d6249a701729d02e28529bf82 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 14:44:53 -0500
Subject: Only Re-Alloc / Reset When Needed When Switching Between Regular and
 High Compression Modes

---
 lib/lz4frame.c | 40 +++++++++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index bf70b5c..cd61925 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -188,7 +188,15 @@ typedef struct LZ4F_cctx_s
     U64    totalInSize;
     XXH32_state_t xxh;
     void*  lz4CtxPtr;
-    U32    lz4CtxLevel;   /* 0: unallocated;  1: LZ4_stream_t;  3: LZ4_streamHC_t */
+    /* lz4CtxLevel records both what size of memory has been allocated into the
+     * ctx pointer field and what it's being used as.
+     * - The low bit (& 1) value indicates whether the ctx is being used for
+     *   hc (1) or not (0).
+     * - The next bit (& 2) indicates whether the allocated memory is big
+     *   enough for a non-hc context.
+     * - The next bit (& 4) indicates whether the allocated memory is big
+     *   enough for an hc context. */
+    U32    lz4CtxLevel;
 } LZ4F_cctx_t;
 
 
@@ -356,7 +364,7 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
     if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
         LZ4_resetStream(&lz4ctx);
         cctxI.lz4CtxPtr = &lz4ctx;
-        cctxI.lz4CtxLevel = 1;
+        cctxI.lz4CtxLevel = 2;
     }  /* fast compression context pre-created on stack */
 
     memset(&options, 0, sizeof(options));
@@ -513,21 +521,27 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->prefs = *preferencesPtr;
 
     /* Ctx Management */
-    {   U32 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;  /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
-        if (cctxPtr->lz4CtxLevel < ctxTypeID) {
+    {   U32 const ctxSufficientAllocBits = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 2 : 6;
+        U32 const ctxTypeBit = cctxPtr->prefs.compressionLevel >= LZ4HC_CLEVEL_MIN;
+        if ((cctxPtr->lz4CtxLevel & ctxSufficientAllocBits) != ctxSufficientAllocBits) {
             FREEMEM(cctxPtr->lz4CtxPtr);
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
-            else
+            } else {
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
+            }
             if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
-            cctxPtr->lz4CtxLevel = ctxTypeID;
-        } else if (cctxPtr->lz4CtxLevel != ctxTypeID) {
-            /* otherwise, we must be transitioning from HC -> LZ4.
-             * In that case, avoid reallocating, since a LZ4 ctx
-             * fits in an HC ctx. Just reset. */
-            LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
-            cctxPtr->lz4CtxLevel = ctxTypeID;
+            cctxPtr->lz4CtxLevel = ctxSufficientAllocBits | ctxTypeBit;
+        } else if ((cctxPtr->lz4CtxLevel & 1) != ctxTypeBit) {
+            /* otherwise, a sufficient buffer is allocated, but we need to
+             * reset it to the correct context type */
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+                LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
+                cctxPtr->lz4CtxLevel &= ~1;
+            } else {
+                LZ4_resetStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+                cctxPtr->lz4CtxLevel |= 1;
+            }
         }
     }
 
-- 
cgit v0.12


From 62cb52b3410bdfd696669910de010ab03666edc8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:10:15 -0500
Subject: Initialize Current Offset to 1

---
 lib/lz4.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d558b3b..dde27e9 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -550,7 +550,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                            tableType == byPtr);
 
     size_t currentOffset = ((tableType == byU32 || tableType == byU16) &&
-                            !resetTable) ? cctx->currentOffset : 0;
+                            !resetTable) ? cctx->currentOffset : 1;
     const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
     const BYTE* const lowRefLimit = ip - cctx->dictSize;
@@ -593,7 +593,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     if (resetTable) {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 0;
+      cctx->currentOffset = 1;
     }
 
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
@@ -793,7 +793,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     int result;
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
-    ctxPtr->internal_donotuse.currentOffset = 0;
+    ctxPtr->internal_donotuse.currentOffset = 1;
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1039,7 +1039,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-    LZ4_stream->internal_donotuse.currentOffset = 0;
+    LZ4_stream->internal_donotuse.currentOffset = 1;
     LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 
-- 
cgit v0.12


From 73cc39327e3abc28a360323c4f26c3c34d87ff07 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:18:24 -0500
Subject: Lookup Matches in Separate Dictionary Context

---
 lib/lz4.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++------------
 lib/lz4.h | 14 +++++++------
 2 files changed, 65 insertions(+), 19 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index dde27e9..40cac94 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -448,7 +448,7 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
 typedef enum { unusedTable = 0, byPtr = 1, byU32 = 2, byU16 = 3 } tableType_t;
 
-typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive;
+typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingExtDictCtx } dict_directive;
 typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
 
 typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
@@ -533,7 +533,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                  const int maxOutputSize,
                  const limitedOutput_directive outputLimited,
                  const tableType_t tableType,
-                 const dict_directive dict,
+                 const dict_directive dictDirective,
                  const dictIssue_directive dictIssue,
                  const U32 acceleration)
 {
@@ -553,15 +553,27 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                             !resetTable) ? cctx->currentOffset : 1;
     const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
-    const BYTE* const lowRefLimit = ip - cctx->dictSize;
-    const BYTE* const dictionary = cctx->dictionary;
-    const BYTE* const dictEnd = dictionary + cctx->dictSize;
-    const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source;
+
+    const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
+    const BYTE* const dictionary =
+        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictionary;
+    const U32 dictSize =
+        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictSize;
+
+    const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
+    const BYTE* const dictEnd = dictionary + dictSize;
     const BYTE* anchor = (const BYTE*) source;
     const BYTE* const iend = ip + inputSize;
     const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1;
     const BYTE* const matchlimit = iend - LASTLITERALS;
 
+    /* the dictCtx currentOffset is indexed on the start of the dictionary,
+     * while a dictionary in the current context precedes the currentOffset */
+    const BYTE* dictBase = dictDirective == usingExtDictCtx ?
+        (const BYTE*) source - dictCtx->currentOffset :
+        (const BYTE*) source - dictSize - currentOffset;
+    const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
+
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
 
@@ -574,17 +586,19 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
-    switch(dict)
+    switch(dictDirective)
     {
     case noDict:
     default:
         lowLimit = (const BYTE*)source;
         break;
     case withPrefix64k:
-        lowLimit = (const BYTE*)source - cctx->dictSize;
+        lowLimit = (const BYTE*)source - dictSize;
         break;
     case usingExtDict:
-        base = (const BYTE*)source - cctx->currentOffset;
+        lowLimit = (const BYTE*)source;
+        break;
+    case usingExtDictCtx:
         lowLimit = (const BYTE*)source;
         break;
     }
@@ -622,8 +636,19 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 assert(ip < mflimitPlusOne);
 
                 match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
-                if (dict==usingExtDict) {
+                if (dictDirective == usingExtDictCtx) {
                     if (match < (const BYTE*)source) {
+                        /* there was no match, try the dictionary */
+                        /* TODO: use precalc-ed hash? */
+                        match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
+                        refDelta = dictDelta;
+                        lowLimit = dictionary;
+                    } else {
+                        refDelta = 0;
+                        lowLimit = (const BYTE*)source;
+                    }
+                } else if (dictDirective==usingExtDict) {
+                    if (match < (const BYTE*)source && dictionary != NULL) {
                         refDelta = dictDelta;
                         lowLimit = dictionary;
                     } else {
@@ -667,7 +692,7 @@ _next_match:
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dict==usingExtDict) && (lowLimit==dictionary)) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictionary)) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
@@ -712,8 +737,19 @@ _next_match:
 
         /* Test next position */
         match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
-        if (dict==usingExtDict) {
+        if (dictDirective == usingExtDictCtx) {
             if (match < (const BYTE*)source) {
+                /* there was no match, try the dictionary */
+                /* TODO: use precalc-ed hash? */
+                match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
+                refDelta = dictDelta;
+                lowLimit = dictionary;
+            } else {
+                refDelta = 0;
+                lowLimit = (const BYTE*)source;
+            }
+        } else if (dictDirective==usingExtDict) {
+            if (match < (const BYTE*)source && dictionary != NULL) {
                 refDelta = dictDelta;
                 lowLimit = dictionary;
             } else {
@@ -751,8 +787,15 @@ _last_literals:
     retval = (((char*)op)-dest);
 
 _clean_up:
+    if (dictDirective == usingExtDictCtx) {
+      /* Subsequent linked blocks can't use the dictionary. */
+      /* Instead, they use the block we just compressed. */
+      cctx->dictCtx = NULL;
+      cctx->dictSize = (U32)inputSize;
+    } else {
+      cctx->dictSize += (U32)inputSize;
+    }
     cctx->currentOffset += (U32)inputSize;
-    cctx->dictSize += (U32)inputSize;
     cctx->tableType = tableType;
 
     /* End */
@@ -773,6 +816,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
     ctx->dictionary = NULL;
     ctx->dictSize = 0;
+    ctx->dictCtx = NULL;
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit)
diff --git a/lib/lz4.h b/lib/lz4.h
index 0013ffe..58a1e39 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -365,15 +365,16 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #include <stdint.h>
 
-typedef struct {
+typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
+struct LZ4_stream_t_internal {
     uint32_t hashTable[LZ4_HASH_SIZE_U32];
     uint32_t currentOffset;
     uint16_t initCheck;
     uint16_t tableType;
     const uint8_t* dictionary;
-    uint8_t* bufferStart;   /* obsolete, used for slideInputBuffer */
+    const LZ4_stream_t_internal* dictCtx;
     uint32_t dictSize;
-} LZ4_stream_t_internal;
+};
 
 typedef struct {
     const uint8_t* externalDict;
@@ -384,15 +385,16 @@ typedef struct {
 
 #else
 
-typedef struct {
+typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
+struct LZ4_stream_t_internal {
     unsigned int hashTable[LZ4_HASH_SIZE_U32];
     unsigned int currentOffset;
     unsigned short initCheck;
     unsigned short tableType;
     const unsigned char* dictionary;
-    unsigned char* bufferStart;   /* obsolete, used for slideInputBuffer */
+    const LZ4_stream_t_internal* dictCtx;
     unsigned int dictSize;
-} LZ4_stream_t_internal;
+};
 
 typedef struct {
     const unsigned char* externalDict;
-- 
cgit v0.12


From 68c6bd17b82a0be501237af682f6472b0d39114f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:19:13 -0500
Subject: Set Dictionary Context Pointer Rather than Copying the Context In

---
 lib/lz4.c      | 17 +++++++++++++----
 lib/lz4frame.c | 35 +++++++++++++++++++++++++++++------
 2 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 40cac94..7d1d65b 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1183,10 +1183,19 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* external dictionary mode */
     {   int result;
-        if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
-        else
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
+        if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
+            if (streamPtr->dictCtx) {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, dictSmall, acceleration);
+            } else {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
+            }
+        } else {
+            if (streamPtr->dictCtx) {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, noDictIssue, acceleration);
+            } else {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
+            }
+        }
         streamPtr->dictionary = (const BYTE*)source;
         streamPtr->dictSize = (U32)inputSize;
         return result;
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index cd61925..9ff7766 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -571,16 +571,36 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         /* frame init only for blockLinked : blockIndependent will be init at each block */
         if (cdict) {
             if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-                memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
+                assert(!internal_ctx->initCheck);
+                if (internal_ctx->currentOffset > 1 GB) {
+                    /* Init the context */
+                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
+                }
+                /* Clear any local dictionary */
+                internal_ctx->dictionary = NULL;
+                internal_ctx->dictSize = 0;
+                /* Point to the dictionary context */
+                internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
             } else {
                 memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
                 LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
             }
         } else {
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-                LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
-            else
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
+                assert(!internal_ctx->initCheck);
+                if (internal_ctx->currentOffset > 1 GB) {
+                    /* Init the context */
+                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
+                }
+                /* Clear any local dictionary */
+                internal_ctx->dictionary = NULL;
+                internal_ctx->dictSize = 0;
+                internal_ctx->dictCtx = NULL;
+            } else {
                 LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
+            }
         }
     }
 
@@ -686,10 +706,13 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize
     internal_ctx->dictionary = NULL;
     internal_ctx->dictSize = 0;
     if (cdict) {
-        memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
+        /* Point to the dictionary context */
+        internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
+    } else {
+        internal_ctx->dictCtx = NULL;
+        return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
-    return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
 static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
-- 
cgit v0.12


From cea09d67a9deca8370dc634fd6c57437114cacbf Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 29 Jan 2018 17:09:52 -0500
Subject: Hoist Table Reset One Level Up

---
 lib/lz4.c | 86 +++++++++++++++++++++++++++++++++++++++------------------------
 1 file changed, 53 insertions(+), 33 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 7d1d65b..1f0450e 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -523,6 +523,29 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
 }
 
 
+LZ4_FORCE_INLINE void LZ4_resetTable(
+        LZ4_stream_t_internal* const cctx,
+        const int inputSize,
+        const tableType_t tableType) {
+  /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
+   * therefore safe to use no matter what mode we're in. Otherwise, we figure
+   * out if it's safe to leave as is or whether it needs to be reset.
+   */
+  if (cctx->tableType != unusedTable && (
+      cctx->tableType != tableType ||
+      (tableType == byU16 &&
+       cctx->currentOffset + inputSize >= 0xFFFFU) ||
+      (tableType == byU32 &&
+       cctx->currentOffset > 1 GB) ||
+      tableType == byPtr))
+  {
+      DEBUGLOG(4, "Resetting table in %p", cctx);
+      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
+      cctx->currentOffset = 1;
+      cctx->tableType = unusedTable;
+  }
+}
+
 /** LZ4_compress_generic() :
     inlined, to ensure branches are decided at compilation time */
 LZ4_FORCE_INLINE int LZ4_compress_generic(
@@ -539,18 +562,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 {
     const BYTE* ip = (const BYTE*) source;
 
-    /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
-     * therefore safe to use no matter what mode we're in. Otherwise, we figure
-     * out if it's safe to leave as is or whether it needs to be reset.
-     */
-    const int resetTable = cctx->tableType != unusedTable && (
-                           cctx->tableType != tableType ||
-                            (tableType == byU16 &&
-                             cctx->currentOffset + inputSize >= 0xFFFFU) ||
-                           tableType == byPtr);
-
-    size_t currentOffset = ((tableType == byU32 || tableType == byU16) &&
-                            !resetTable) ? cctx->currentOffset : 1;
+    size_t currentOffset = cctx->currentOffset;
     const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
 
@@ -604,12 +616,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     }
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
 
-    if (resetTable) {
-      DEBUGLOG(4, "Resetting table in %p", cctx);
-      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 1;
-    }
-
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* First Byte */
@@ -819,15 +825,25 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     ctx->dictCtx = NULL;
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
-        if (inputSize < LZ4_64Klimit)
-            return LZ4_compress_generic(ctx, source, dest, inputSize,             0,    notLimited,                        byU16, noDict, noDictIssue, acceleration);
-        else
-            return LZ4_compress_generic(ctx, source, dest, inputSize,             0,    notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
+        if (inputSize < LZ4_64Klimit) {
+            const tableType_t tableType = byU16;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+        }
     } else {
-        if (inputSize < LZ4_64Klimit)
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput,                        byU16, noDict, noDictIssue, acceleration);
-        else
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
+        if (inputSize < LZ4_64Klimit) {
+            const tableType_t tableType = byU16;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+        }
     }
 }
 
@@ -842,7 +858,8 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
 #endif
-    LZ4_resetStream(ctxPtr);
+    ctxPtr->internal_donotuse.initCheck = 0;
+    ctxPtr->internal_donotuse.tableType = byPtr; /* always triggers a reset */
     result = LZ4_compress_fast_safeExtState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
 
 #if (LZ4_HEAPMODE)
@@ -1154,6 +1171,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
 
 int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
+    const tableType_t tableType = byU32;
     LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
     const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
@@ -1173,27 +1191,29 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
         }
     }
 
+    LZ4_resetTable(streamPtr, inputSize, tableType);
+
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
-            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
-            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration);
     }
 
     /* external dictionary mode */
     {   int result;
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
             if (streamPtr->dictCtx) {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, dictSmall, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
             } else {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             }
         } else {
             if (streamPtr->dictCtx) {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             } else {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             }
         }
         streamPtr->dictionary = (const BYTE*)source;
-- 
cgit v0.12


From 14ce912b705db03bd1d723456dd1a3242b7f7ea7 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Feb 2018 13:46:36 -0800
Subject: Switch Current Offset to 1 Only When in External Dictionary Context
 Mode

---
 lib/lz4.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 1f0450e..cace35f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -526,7 +526,8 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
 LZ4_FORCE_INLINE void LZ4_resetTable(
         LZ4_stream_t_internal* const cctx,
         const int inputSize,
-        const tableType_t tableType) {
+        const tableType_t tableType,
+        const dict_directive dictDirective) {
   /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
    * therefore safe to use no matter what mode we're in. Otherwise, we figure
    * out if it's safe to leave as is or whether it needs to be reset.
@@ -541,9 +542,12 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
   {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 1;
+      cctx->currentOffset = 0;
       cctx->tableType = unusedTable;
   }
+  if (dictDirective == usingExtDictCtx && cctx->currentOffset == 0) {
+      cctx->currentOffset = 1;
+  }
 }
 
 /** LZ4_compress_generic() :
@@ -827,21 +831,21 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
@@ -853,7 +857,6 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     int result;
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
-    ctxPtr->internal_donotuse.currentOffset = 1;
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1100,7 +1103,6 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-    LZ4_stream->internal_donotuse.currentOffset = 1;
     LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 
@@ -1191,10 +1193,10 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
         }
     }
 
-    LZ4_resetTable(streamPtr, inputSize, tableType);
 
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
+        LZ4_resetTable(streamPtr, inputSize, tableType, withPrefix64k);
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
             return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
@@ -1205,14 +1207,18 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
     {   int result;
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
             if (streamPtr->dictCtx) {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
             } else {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             }
         } else {
             if (streamPtr->dictCtx) {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             } else {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             }
         }
-- 
cgit v0.12


From 9dcd9abc14a33b6ac6c91dd727235db1daabe066 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Jan 2018 11:29:00 -0500
Subject: Make LZ4F_compressFrame_usingCDict Take a Compression Context

---
 lib/lz4frame.c        | 75 ++++++++++++++++++++++++++++++++++++---------------
 lib/lz4frame_static.h |  2 ++
 programs/lz4io.c      |  2 +-
 tests/framebench.c    |  2 ++
 tests/frametest.c     | 19 +++++++------
 5 files changed, 69 insertions(+), 31 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 9ff7766..e77fd35 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -47,6 +47,19 @@ You can contact the author at :
 
 
 /*-************************************
+*  Tuning parameters
+**************************************/
+/*
+ * LZ4_HEAPMODE :
+ * Select how default compression functions will allocate memory for their hash table,
+ * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
+ */
+#ifndef LZ4_HEAPMODE
+#  define LZ4_HEAPMODE 0
+#endif
+
+
+/*-************************************
 *  Memory routines
 **************************************/
 #include <stdlib.h>   /* malloc, calloc, free */
@@ -332,23 +345,18 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
  * @return : number of bytes written into dstBuffer,
  *           or an error code if it fails (can be tested using LZ4F_isError())
  */
-size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
+size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
+                                     void* dstBuffer, size_t dstCapacity,
                                const void* srcBuffer, size_t srcSize,
                                const LZ4F_CDict* cdict,
                                const LZ4F_preferences_t* preferencesPtr)
 {
-    LZ4F_cctx_t cctxI;
-    LZ4_stream_t lz4ctx;   /* pretty large on stack */
     LZ4F_preferences_t prefs;
     LZ4F_compressOptions_t options;
     BYTE* const dstStart = (BYTE*) dstBuffer;
     BYTE* dstPtr = dstStart;
     BYTE* const dstEnd = dstStart + dstCapacity;
 
-    memset(&cctxI, 0, sizeof(cctxI));
-    cctxI.version = LZ4F_VERSION;
-    cctxI.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
-
     if (preferencesPtr!=NULL)
         prefs = *preferencesPtr;
     else
@@ -361,33 +369,24 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
     if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
         prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* only one block => no need for inter-block link */
 
-    if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-        LZ4_resetStream(&lz4ctx);
-        cctxI.lz4CtxPtr = &lz4ctx;
-        cctxI.lz4CtxLevel = 2;
-    }  /* fast compression context pre-created on stack */
-
     memset(&options, 0, sizeof(options));
     options.stableSrc = 1;
 
     if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs))  /* condition to guarantee success */
         return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
 
-    { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs);  /* write header */
+    { size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs);  /* write header */
       if (LZ4F_isError(headerSize)) return headerSize;
       dstPtr += headerSize;   /* header size */ }
 
-    { size_t const cSize = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
+    { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
       if (LZ4F_isError(cSize)) return cSize;
       dstPtr += cSize; }
 
-    { size_t const tailSize = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options);   /* flush last block, and generate suffix */
+    { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, &options);   /* flush last block, and generate suffix */
       if (LZ4F_isError(tailSize)) return tailSize;
       dstPtr += tailSize; }
 
-    if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN)  /* Ctx allocation only for lz4hc */
-        FREEMEM(cctxI.lz4CtxPtr);
-
     return (dstPtr - dstStart);
 }
 
@@ -403,9 +402,41 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
                     const void* srcBuffer, size_t srcSize,
                     const LZ4F_preferences_t* preferencesPtr)
 {
-    return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity,
-                                         srcBuffer, srcSize,
-                                         NULL, preferencesPtr);
+    size_t result;
+#if (LZ4_HEAPMODE)
+    LZ4F_cctx_t *cctxPtr;
+    LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
+#else
+    LZ4F_cctx_t cctx;
+    LZ4_stream_t lz4ctx;
+    LZ4F_cctx_t *cctxPtr = &cctx;
+
+    MEM_INIT(&cctx, 0, sizeof(cctx));
+    cctx.version = LZ4F_VERSION;
+    cctx.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
+    if (preferencesPtr == NULL ||
+        preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN
+    ) {
+        LZ4_resetStream(&lz4ctx);
+        cctxPtr->lz4CtxPtr = &lz4ctx;
+        cctxPtr->lz4CtxLevel = 2;
+    }
+#endif
+
+    result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity,
+                                           srcBuffer, srcSize,
+                                           NULL, preferencesPtr);
+
+#if (LZ4_HEAPMODE)
+    LZ4F_freeCompressionContext(cctxPtr);
+#else
+    if (preferencesPtr != NULL &&
+        preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN
+    ) {
+        FREEMEM(cctxPtr->lz4CtxPtr);
+    }
+#endif
+    return result;
 }
 
 
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index a59b94b..be587e6 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -107,6 +107,7 @@ LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
 
 /*! LZ4_compressFrame_usingCDict() :
  *  Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+ *  cctx must point to a context created by LZ4F_createCompressionContext().
  *  If cdict==NULL, compress without a dictionary.
  *  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
  *  If this condition is not respected, function will fail (@return an errorCode).
@@ -115,6 +116,7 @@ LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
  * @return : number of bytes written into dstBuffer.
  *           or an error code if it fails (can be tested using LZ4F_isError()) */
 LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
+    LZ4F_cctx* cctx,
     void* dst, size_t dstCapacity,
     const void* src, size_t srcSize,
     const LZ4F_CDict* cdict,
diff --git a/programs/lz4io.c b/programs/lz4io.c
index c712fe1..ca13316 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -560,7 +560,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
     /* single-block file */
     if (readSize < blockSize) {
         /* Compress in single pass */
-        size_t cSize = LZ4F_compressFrame_usingCDict(dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
+        size_t cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
         if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
         compressedfilesize = cSize;
         DISPLAYUPDATE(2, "\rRead : %u MB   ==> %.2f%%   ",
diff --git a/tests/framebench.c b/tests/framebench.c
index 21c3704..a7a270b 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -30,6 +30,7 @@ typedef struct {
 
 size_t compress_frame(bench_params_t *p) {
   size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
   const char* ibuf = p->ibuf;
@@ -43,6 +44,7 @@ size_t compress_frame(bench_params_t *p) {
   prefs->frameInfo.contentSize = isize;
 
   oused = LZ4F_compressFrame_usingCDict(
+    cctx,
     obuf,
     osize,
     ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
diff --git a/tests/frametest.c b/tests/frametest.c
index 986a16d..74d9c88 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -164,7 +164,7 @@ static unsigned FUZ_highbit(U32 v32)
 /*-*******************************************************
 *  Tests
 *********************************************************/
-#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) goto _output_error
+#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) { fprintf(stderr, "%s\n", LZ4F_getErrorName(v)); goto _output_error; }
 #define CHECK(f)   { LZ4F_errorCode_t const CHECK_V(err_ , f); }
 
 int basicTests(U32 seed, double compressibility)
@@ -509,23 +509,25 @@ int basicTests(U32 seed, double compressibility)
         CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
     }
 
-
     /* Dictionary compression test */
     {   size_t const dictSize = 63 KB;
         size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL);
         size_t cSizeNoDict, cSizeWithDict;
         LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
         if (cdict == NULL) goto _output_error;
+        CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
         CHECK_V(cSizeNoDict,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               NULL, NULL) );
         DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
 
+        CHECK( LZ4F_freeCompressionContext(cctx) );
+        CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
         CHECK_V(cSizeWithDict,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               cdict, NULL) );
         DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
@@ -557,7 +559,7 @@ int basicTests(U32 seed, double compressibility)
             memset(&cParams, 0, sizeof(cParams));
             cParams.compressionLevel = -3;
             CHECK_V(cSizeLevelMax,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
@@ -569,7 +571,7 @@ int basicTests(U32 seed, double compressibility)
             memset(&cParams, 0, sizeof(cParams));
             cParams.compressionLevel = LZ4F_compressionLevel_max();
             CHECK_V(cSizeLevelMax,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
@@ -584,7 +586,7 @@ int basicTests(U32 seed, double compressibility)
             cParams.frameInfo.blockMode = LZ4F_blockLinked;
             cParams.frameInfo.blockSizeID = LZ4F_max64KB;
             CHECK_V(cSizeContiguous,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
                                               CNBuffer, inSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
@@ -620,7 +622,7 @@ int basicTests(U32 seed, double compressibility)
             cParams.frameInfo.blockMode = LZ4F_blockIndependent;
             cParams.frameInfo.blockSizeID = LZ4F_max64KB;
             CHECK_V(cSizeIndep,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
                                               CNBuffer, inSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
@@ -647,6 +649,7 @@ int basicTests(U32 seed, double compressibility)
         }
 
         LZ4F_freeCDict(cdict);
+        CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
     }
 
 
-- 
cgit v0.12


From 80790c587b55b8e0b932b64c892ba984cde1b70f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Jan 2018 12:06:43 -0500
Subject: Copy the Dict Table Into the Context for Large Compressions

---
 lib/lz4.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index cace35f..f7de564 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1205,20 +1205,25 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* external dictionary mode */
     {   int result;
-        if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-            if (streamPtr->dictCtx) {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
+        if (streamPtr->dictCtx && inputSize < 2 KB) {
+            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
+            if ((streamPtr->dictCtx->dictSize < 64 KB) && (streamPtr->dictCtx->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
             } else {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             }
         } else {
             if (streamPtr->dictCtx) {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
+              /* For compressing large blobs, it is faster to pay the setup
+               * cost to copy the dictionary's tables into the active context,
+               * so that the compression loop is only looking in one table.
+               */
+              memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
+            }
+            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
+            if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             } else {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             }
         }
-- 
cgit v0.12


From efc419a6d4448a5806715e967384ea85f880fe59 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Jan 2018 17:29:50 -0500
Subject: Replace calloc() Calls With malloc() Where Possible

---
 lib/lz4.c      | 12 ++++++------
 lib/lz4frame.c | 31 ++++++++++++++++---------------
 lib/lz4hc.c    |  2 +-
 3 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index f7de564..990096c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -154,7 +154,8 @@
 *  Memory routines
 **************************************/
 #include <stdlib.h>   /* malloc, calloc, free */
-#define ALLOCATOR(n,s) calloc(n,s)
+#define ALLOC(s) malloc(s)
+#define ALLOC_AND_ZERO(s) calloc(1,s)
 #define FREEMEM        free
 #include <string.h>   /* memset, memcpy */
 #define MEM_INIT       memset
@@ -856,7 +857,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
 {
     int result;
 #if (LZ4_HEAPMODE)
-    LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1070,7 +1071,7 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
 int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)
 {
 #if (LZ4_HEAPMODE)
-    LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
 #else
     LZ4_stream_t ctxBody;
     LZ4_stream_t* ctx = &ctxBody;
@@ -1092,7 +1093,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
 
 LZ4_stream_t* LZ4_createStream(void)
 {
-    LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64);
+    LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));
     LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal));    /* A compilation error here means LZ4_STREAMSIZE is not large enough */
     DEBUGLOG(4, "LZ4_createStream %p", lz4s);
     LZ4_resetStream(lz4s);
@@ -1193,7 +1194,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
         }
     }
 
-
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
         LZ4_resetTable(streamPtr, inputSize, tableType, withPrefix64k);
@@ -1493,7 +1493,7 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
 
 LZ4_streamDecode_t* LZ4_createStreamDecode(void)
 {
-    LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t));
+    LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t));
     return lz4s;
 }
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index e77fd35..8ed89e3 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -63,7 +63,8 @@ You can contact the author at :
 *  Memory routines
 **************************************/
 #include <stdlib.h>   /* malloc, calloc, free */
-#define ALLOCATOR(s)   calloc(1,s)
+#define ALLOC(s)   malloc(s)
+#define ALLOC_AND_ZERO(s)   calloc(1,s)
 #define FREEMEM        free
 #include <string.h>   /* memset, memcpy, memmove */
 #define MEM_INIT       memset
@@ -300,7 +301,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
                                           size_t alreadyBuffered)
 {
     LZ4F_preferences_t prefsNull;
-    memset(&prefsNull, 0, sizeof(prefsNull));
+    MEM_INIT(&prefsNull, 0, sizeof(prefsNull));
     prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;   /* worst case */
     {   const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
         U32 const flush = prefsPtr->autoFlush | (srcSize==0);
@@ -329,7 +330,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
     size_t const headerSize = maxFHSize;      /* max header size, including optional fields */
 
     if (preferencesPtr!=NULL) prefs = *preferencesPtr;
-    else memset(&prefs, 0, sizeof(prefs));
+    else MEM_INIT(&prefs, 0, sizeof(prefs));
     prefs.autoFlush = 1;
 
     return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);;
@@ -360,7 +361,7 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
     if (preferencesPtr!=NULL)
         prefs = *preferencesPtr;
     else
-        memset(&prefs, 0, sizeof(prefs));
+        MEM_INIT(&prefs, 0, sizeof(prefs));
     if (prefs.frameInfo.contentSize != 0)
         prefs.frameInfo.contentSize = (U64)srcSize;   /* auto-correct content size if selected (!=0) */
 
@@ -369,7 +370,7 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
     if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
         prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* only one block => no need for inter-block link */
 
-    memset(&options, 0, sizeof(options));
+    MEM_INIT(&options, 0, sizeof(options));
     options.stableSrc = 1;
 
     if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs))  /* condition to guarantee success */
@@ -465,7 +466,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
         dictStart += dictSize - 64 KB;
         dictSize = 64 KB;
     }
-    cdict->dictContent = ALLOCATOR(dictSize);
+    cdict->dictContent = ALLOC(dictSize);
     cdict->fastCtx = LZ4_createStream();
     cdict->HCCtx = LZ4_createStreamHC();
     if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
@@ -504,7 +505,7 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict)
  */
 LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
 {
-    LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t));
+    LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t));
     if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
 
     cctxPtr->version = version;
@@ -547,7 +548,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     BYTE* headerStart;
 
     if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
-    memset(&prefNull, 0, sizeof(prefNull));
+    MEM_INIT(&prefNull, 0, sizeof(prefNull));
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
     cctxPtr->prefs = *preferencesPtr;
 
@@ -588,7 +589,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         if (cctxPtr->maxBufferSize < requiredBuffSize) {
             cctxPtr->maxBufferSize = 0;
             FREEMEM(cctxPtr->tmpBuff);
-            cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
+            cctxPtr->tmpBuff = (BYTE*)ALLOC(requiredBuffSize);
             if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             cctxPtr->maxBufferSize = requiredBuffSize;
     }   }
@@ -813,7 +814,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
         return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
-    memset(&cOptionsNull, 0, sizeof(cOptionsNull));
+    MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull));
     if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
 
     /* complete tmp buffer */
@@ -1017,7 +1018,7 @@ struct LZ4F_dctx_s {
  */
 LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
 {
-    LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
+    LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx));
     if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC);
 
     dctx->version = versionNumber;
@@ -1089,7 +1090,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
 
     /* need to decode header to get frameInfo */
     if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete);   /* minimal frame header size */
-    memset(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
+    MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
 
     /* special case : skippable frames */
     if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) {
@@ -1316,7 +1317,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
     size_t nextSrcSizeHint = 1;
 
 
-    memset(&optionsNull, 0, sizeof(optionsNull));
+    MEM_INIT(&optionsNull, 0, sizeof(optionsNull));
     if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull;
     *srcSizePtr = 0;
     *dstSizePtr = 0;
@@ -1365,11 +1366,11 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 if (bufferNeeded > dctx->maxBufferSize) {   /* tmp buffers too small */
                     dctx->maxBufferSize = 0;   /* ensure allocation will be re-attempted on next entry*/
                     FREEMEM(dctx->tmpIn);
-                    dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize + 4 /* block checksum */);
+                    dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + 4 /* block checksum */);
                     if (dctx->tmpIn == NULL)
                         return err0r(LZ4F_ERROR_allocation_failed);
                     FREEMEM(dctx->tmpOutBuffer);
-                    dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
+                    dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded);
                     if (dctx->tmpOutBuffer== NULL)
                         return err0r(LZ4F_ERROR_allocation_failed);
                     dctx->maxBufferSize = bufferNeeded;
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 2a6e080..0c1b20a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -858,7 +858,7 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
 
 void* LZ4_createHC (char* inputBuffer)
 {
-    LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t));
+    LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
     if (hc4 == NULL) return NULL;   /* not enough memory */
     LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
     hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer;
-- 
cgit v0.12


From d6ed9a7799c35d9d259af36c3142739216ba8a58 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 30 Jan 2018 15:22:29 -0500
Subject: Avoid dictionary == NULL Check

---
 lib/lz4.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 990096c..3e456ac 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -590,6 +590,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         (const BYTE*) source - dictCtx->currentOffset :
         (const BYTE*) source - dictSize - currentOffset;
     const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
+    const BYTE* dictLowLimit;
 
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
@@ -619,6 +620,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         lowLimit = (const BYTE*)source;
         break;
     }
+
+    dictLowLimit = dictionary ? dictionary : lowLimit;
+
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
 
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
@@ -653,15 +657,15 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                         /* TODO: use precalc-ed hash? */
                         match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                         refDelta = dictDelta;
-                        lowLimit = dictionary;
+                        lowLimit = dictLowLimit;
                     } else {
                         refDelta = 0;
                         lowLimit = (const BYTE*)source;
                     }
                 } else if (dictDirective==usingExtDict) {
-                    if (match < (const BYTE*)source && dictionary != NULL) {
+                    if (match < (const BYTE*)source) {
                         refDelta = dictDelta;
-                        lowLimit = dictionary;
+                        lowLimit = dictLowLimit;
                     } else {
                         refDelta = 0;
                         lowLimit = (const BYTE*)source;
@@ -703,7 +707,7 @@ _next_match:
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictionary)) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictLowLimit)) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
@@ -754,15 +758,15 @@ _next_match:
                 /* TODO: use precalc-ed hash? */
                 match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                 refDelta = dictDelta;
-                lowLimit = dictionary;
+                lowLimit = dictLowLimit;
             } else {
                 refDelta = 0;
                 lowLimit = (const BYTE*)source;
             }
         } else if (dictDirective==usingExtDict) {
-            if (match < (const BYTE*)source && dictionary != NULL) {
+            if (match < (const BYTE*)source) {
                 refDelta = dictDelta;
-                lowLimit = dictionary;
+                lowLimit = dictLowLimit;
             } else {
                 refDelta = 0;
                 lowLimit = (const BYTE*)source;
-- 
cgit v0.12


From 1c4601d64377cdd4d5fc17ccd1f6e1426c7ddd04 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 16 Feb 2018 17:33:51 -0800
Subject: Restore DictIssue Check

---
 lib/lz4.c | 37 +++++++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 3e456ac..a401ee2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -599,9 +599,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     U32 forwardH;
 
-    /* TODO: resurrect dictIssue check */
-    (void)dictIssue;
-
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     switch(dictDirective)
@@ -673,7 +670,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
 
-            } while ( match < lowRefLimit
+            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
                 || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
         }
@@ -772,7 +769,7 @@ _next_match:
                 lowLimit = (const BYTE*)source;
         }   }
         LZ4_putPosition(ip, cctx->hashTable, tableType, base);
-        if ( match >= lowRefLimit
+        if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
             && (match+MAX_DISTANCE>=ip)
             && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
         { token=op++; *token=0; goto _next_match; }
@@ -837,21 +834,37 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            }
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            }
         }
     }
 }
@@ -1249,7 +1262,11 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
     LZ4_renormDictT(streamPtr, smallest);
 
-    result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+    if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
+      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+    } else {
+      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+    }
 
     streamPtr->dictionary = (const BYTE*)source;
     streamPtr->dictSize = (U32)inputSize;
-- 
cgit v0.12


From d571d0cdbaa7032fff48543f272f6d5472041aa7 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 5 Mar 2018 11:59:22 -0500
Subject: Avoid DictSmall Checks By Strategically Bumping CurrentOffset

---
 lib/lz4.c | 45 +++++++++++++++++++++++----------------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index a401ee2..7c5b444 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -546,7 +546,10 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
       cctx->currentOffset = 0;
       cctx->tableType = unusedTable;
   }
-  if (dictDirective == usingExtDictCtx && cctx->currentOffset == 0) {
+  if (dictDirective == usingExtDictCtx &&
+      tableType != byPtr &&
+      cctx->currentOffset == 0)
+  {
       cctx->currentOffset = 1;
   }
 }
@@ -842,11 +845,8 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
-            } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
-            }
+            ctx->currentOffset += 64 KB;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
@@ -860,11 +860,8 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
-            } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
-            }
+            ctx->currentOffset += 64 KB;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
 }
@@ -1222,21 +1219,25 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* external dictionary mode */
     {   int result;
-        if (streamPtr->dictCtx && inputSize < 2 KB) {
-            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
-            if ((streamPtr->dictCtx->dictSize < 64 KB) && (streamPtr->dictCtx->dictSize < streamPtr->currentOffset)) {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
+        if (streamPtr->dictCtx) {
+            /* We depend here on the fact that dictCtx'es (produced by
+             * LZ4_loadDict) guarantee that their tables contain no references
+             * to offsets between dictCtx->currentOffset - 64 KB and
+             * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe
+             * to use noDictIssue even when the dict isn't a full 64 KB.
+             */
+            if (inputSize > 2 KB) {
+                /* For compressing large blobs, it is faster to pay the setup
+                 * cost to copy the dictionary's tables into the active context,
+                 * so that the compression loop is only looking in one table.
+                 */
+                memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             }
         } else {
-            if (streamPtr->dictCtx) {
-              /* For compressing large blobs, it is faster to pay the setup
-               * cost to copy the dictionary's tables into the active context,
-               * so that the compression loop is only looking in one table.
-               */
-              memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
-            }
             LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
-- 
cgit v0.12


From 00eadadbfc37587e88733942a252c2f106c67ef2 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Feb 2018 17:06:24 -0800
Subject: Reset Table on Inputs Larger than 2KB

---
 lib/lz4.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 7c5b444..4810910 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -539,7 +539,8 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
        cctx->currentOffset + inputSize >= 0xFFFFU) ||
       (tableType == byU32 &&
        cctx->currentOffset > 1 GB) ||
-      tableType == byPtr))
+      tableType == byPtr ||
+      inputSize >= 2 KB))
   {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-- 
cgit v0.12


From b78cf67c9624a6a826ceaa2a9d65f5feb1dd0cfd Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 6 Mar 2018 11:52:02 -0500
Subject: Move to 4KB Cut-Off

---
 lib/lz4.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4810910..d147681 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -540,7 +540,7 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
       (tableType == byU32 &&
        cctx->currentOffset > 1 GB) ||
       tableType == byPtr ||
-      inputSize >= 2 KB))
+      inputSize >= 4 KB))
   {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
@@ -1227,7 +1227,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
              * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe
              * to use noDictIssue even when the dict isn't a full 64 KB.
              */
-            if (inputSize > 2 KB) {
+            if (inputSize > 4 KB) {
                 /* For compressing large blobs, it is faster to pay the setup
                  * cost to copy the dictionary's tables into the active context,
                  * so that the compression loop is only looking in one table.
-- 
cgit v0.12


From 64bcbf400ea6c955b029f74e1214dcd03b90f934 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 6 Mar 2018 15:53:22 -0500
Subject: Optimize Dict Check Condition

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d147681..4f98efa 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -708,7 +708,7 @@ _next_match:
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictLowLimit)) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && ((lowLimit==dictLowLimit) & (dictionary != NULL))) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
-- 
cgit v0.12


From b4335a6585fe55aa81bcadaa29e265ffe9550ff2 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 8 Mar 2018 12:29:45 -0500
Subject: Further Avoid a dictionary==NULL Check

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4f98efa..da68ddd 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -708,7 +708,7 @@ _next_match:
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && ((lowLimit==dictLowLimit) & (dictionary != NULL))) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && lowLimit==dictionary) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
-- 
cgit v0.12


From 6716325ae8db3f177630c875622401e35e4a84e5 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 8 Mar 2018 12:30:34 -0500
Subject: Remove Switch In Favor of Ternary Statement

---
 lib/lz4.c | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index da68ddd..809faf9 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -605,27 +605,11 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
-    switch(dictDirective)
-    {
-    case noDict:
-    default:
-        lowLimit = (const BYTE*)source;
-        break;
-    case withPrefix64k:
-        lowLimit = (const BYTE*)source - dictSize;
-        break;
-    case usingExtDict:
-        lowLimit = (const BYTE*)source;
-        break;
-    case usingExtDictCtx:
-        lowLimit = (const BYTE*)source;
-        break;
-    }
 
+    lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
     dictLowLimit = dictionary ? dictionary : lowLimit;
 
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
-
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* First Byte */
-- 
cgit v0.12


From 5f8967b98364629310d0013b356070ab445f9e66 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 9 Mar 2018 12:14:42 -0500
Subject: Specialize _extState() for Clean Ctx Rather Than Calling
 _safeExtState()

---
 lib/lz4.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 809faf9..82c0ae3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -805,8 +805,24 @@ _clean_up:
 
 int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
-  LZ4_resetStream((LZ4_stream_t*)state);
-  return LZ4_compress_fast_safeExtState(state, source, dest, inputSize, maxOutputSize, acceleration);
+    LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
+    if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
+    LZ4_resetStream((LZ4_stream_t*)state);
+    if (maxOutputSize >= LZ4_compressBound(inputSize)) {
+        if (inputSize < LZ4_64Klimit) {
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, byU16, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+        }
+    } else {
+        if (inputSize < LZ4_64Klimit) {;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+        }
+    }
 }
 
 
@@ -861,9 +877,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
 #endif
-    ctxPtr->internal_donotuse.initCheck = 0;
-    ctxPtr->internal_donotuse.tableType = byPtr; /* always triggers a reset */
-    result = LZ4_compress_fast_safeExtState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
+    result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
 
 #if (LZ4_HEAPMODE)
     FREEMEM(ctxPtr);
-- 
cgit v0.12


From e34716cc3f54d3fca1cbd797e8af003cd3a63bc8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 8 Mar 2018 14:09:06 -0500
Subject: Preserve currentOffset==0 When Possible

---
 lib/lz4.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 82c0ae3..1ea8ba1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -846,7 +846,9 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            ctx->currentOffset += 64 KB;
+            if (ctx->currentOffset) {
+              ctx->currentOffset += 64 KB;
+            }
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
@@ -861,7 +863,9 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            ctx->currentOffset += 64 KB;
+            if (ctx->currentOffset) {
+              ctx->currentOffset += 64 KB;
+            }
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
-- 
cgit v0.12


From 3ecc1d7a5b0f911c57851cc8a925d7be2518d358 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 16:11:44 -0400
Subject: Minor Style Fixes

---
 lib/lz4frame.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 8ed89e3..16769fe 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -50,12 +50,12 @@ You can contact the author at :
 *  Tuning parameters
 **************************************/
 /*
- * LZ4_HEAPMODE :
+ * LZ4F_HEAPMODE :
  * Select how default compression functions will allocate memory for their hash table,
  * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
  */
-#ifndef LZ4_HEAPMODE
-#  define LZ4_HEAPMODE 0
+#ifndef LZ4F_HEAPMODE
+#  define LZ4F_HEAPMODE 0
 #endif
 
 
@@ -404,7 +404,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
                     const LZ4F_preferences_t* preferencesPtr)
 {
     size_t result;
-#if (LZ4_HEAPMODE)
+#if (LZ4F_HEAPMODE)
     LZ4F_cctx_t *cctxPtr;
     LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
 #else
@@ -416,8 +416,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     cctx.version = LZ4F_VERSION;
     cctx.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
     if (preferencesPtr == NULL ||
-        preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN
-    ) {
+        preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN)
+    {
         LZ4_resetStream(&lz4ctx);
         cctxPtr->lz4CtxPtr = &lz4ctx;
         cctxPtr->lz4CtxLevel = 2;
@@ -428,12 +428,12 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
                                            srcBuffer, srcSize,
                                            NULL, preferencesPtr);
 
-#if (LZ4_HEAPMODE)
+#if (LZ4F_HEAPMODE)
     LZ4F_freeCompressionContext(cctxPtr);
 #else
     if (preferencesPtr != NULL &&
-        preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN
-    ) {
+        preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN)
+    {
         FREEMEM(cctxPtr->lz4CtxPtr);
     }
 #endif
-- 
cgit v0.12


From 1df5d911aa23e645b873164382a40186ed704bac Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 16:11:55 -0400
Subject: Hoist LZ4F Dictionary Setup into Helper LZ4F_applyCDict()

---
 lib/lz4frame.c | 72 ++++++++++++++++++++--------------------------------------
 1 file changed, 25 insertions(+), 47 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 16769fe..4cc2ef3 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -531,6 +531,28 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 }
 
 
+static void LZ4F_applyCDict(void* ctx,
+                            const LZ4F_CDict* cdict,
+                            int level) {
+    if (level < LZ4HC_CLEVEL_MIN) {
+        LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
+        assert(!internal_ctx->initCheck);
+        /* Clear any local dictionary */
+        internal_ctx->dictionary = NULL;
+        internal_ctx->dictSize = 0;
+        /* Point to the dictionary context */
+        internal_ctx->dictCtx = cdict ? &(cdict->fastCtx->internal_donotuse) : NULL;
+    } else {
+        if (cdict) {
+            memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
+            LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
+        } else {
+            LZ4_resetStreamHC((LZ4_streamHC_t*)(ctx), level);
+        }
+    }
+}
+
+
 /*! LZ4F_compressBegin_usingCDict() :
  *  init streaming compression and writes frame header into dstBuffer.
  *  dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
@@ -601,39 +623,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->cdict = cdict;
     if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
         /* frame init only for blockLinked : blockIndependent will be init at each block */
-        if (cdict) {
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
-                assert(!internal_ctx->initCheck);
-                if (internal_ctx->currentOffset > 1 GB) {
-                    /* Init the context */
-                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
-                }
-                /* Clear any local dictionary */
-                internal_ctx->dictionary = NULL;
-                internal_ctx->dictSize = 0;
-                /* Point to the dictionary context */
-                internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
-            } else {
-                memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
-                LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
-            }
-        } else {
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
-                assert(!internal_ctx->initCheck);
-                if (internal_ctx->currentOffset > 1 GB) {
-                    /* Init the context */
-                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
-                }
-                /* Clear any local dictionary */
-                internal_ctx->dictionary = NULL;
-                internal_ctx->dictSize = 0;
-                internal_ctx->dictCtx = NULL;
-            } else {
-                LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
-            }
-        }
+        LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
     }
 
     /* Magic Number */
@@ -728,21 +718,10 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
-    LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)ctx)->internal_donotuse;
-    assert(!internal_ctx->initCheck);
-    if (internal_ctx->currentOffset > 1 GB) {
-        /* Init the context */
-        LZ4_resetStream((LZ4_stream_t*)ctx);
-    }
-    /* Clear any local dictionary */
-    internal_ctx->dictionary = NULL;
-    internal_ctx->dictSize = 0;
+    LZ4F_applyCDict(ctx, cdict, level);
     if (cdict) {
-        /* Point to the dictionary context */
-        internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
-        internal_ctx->dictCtx = NULL;
         return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
 }
@@ -757,8 +736,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     if (cdict) {
-        memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
-        LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
+        LZ4F_applyCDict(ctx, cdict, level);
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }
     return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
-- 
cgit v0.12


From f9fef255a12470b7beeaa8635b65fc837d835c84 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 18:13:24 -0400
Subject: Renames and Comment Fixes

---
 lib/lz4.c          | 32 ++++++++++++++++++++++----------
 lib/lz4.h          | 14 +++++++-------
 lib/lz4frame.c     |  2 +-
 tests/framebench.c |  2 +-
 4 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 1ea8ba1..94248be 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -524,7 +524,7 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
 }
 
 
-LZ4_FORCE_INLINE void LZ4_resetTable(
+LZ4_FORCE_INLINE void LZ4_prepareTable(
         LZ4_stream_t_internal* const cctx,
         const int inputSize,
         const tableType_t tableType,
@@ -547,6 +547,11 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
       cctx->currentOffset = 0;
       cctx->tableType = unusedTable;
   }
+  /* If the current offset is zero, we will never look in the external
+   * dictionary context, since there is no value a table entry can take that
+   * indicates a miss. In that case, we need to bump the offset to something
+   * non-zero.
+   */
   if (dictDirective == usingExtDictCtx &&
       tableType != byPtr &&
       cctx->currentOffset == 0)
@@ -825,8 +830,15 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
     }
 }
 
-
-int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+/**
+ * LZ4_compress_fast_extState_noReset is a variant of LZ4_compress_fast_extState
+ * that can be used when the state is known to have already been initialized
+ * (via LZ4_resetStream or an earlier call to LZ4_compress_fast_extState /
+ * LZ4_compress_fast_extState_noReset). This can provide significantly better
+ * performance when the context reset would otherwise be a significant part of
+ * the cost of the compression, e.g., when the data to be compressed is small.
+ */
+int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -837,7 +849,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -845,7 +857,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               ctx->currentOffset += 64 KB;
             }
@@ -854,7 +866,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -862,7 +874,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               ctx->currentOffset += 64 KB;
             }
@@ -1213,7 +1225,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
-        LZ4_resetTable(streamPtr, inputSize, tableType, withPrefix64k);
+        LZ4_prepareTable(streamPtr, inputSize, tableType, withPrefix64k);
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
             return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
@@ -1237,11 +1249,11 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
+                LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             }
         } else {
-            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
+            LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             } else {
diff --git a/lib/lz4.h b/lib/lz4.h
index 58a1e39..ca9d552 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -179,21 +179,21 @@ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int d
 
 
 /*!
-LZ4_compress_fast_safeExtState() :
+LZ4_compress_fast_extState_noReset() :
 LZ4_compress_fast_extState() :
     Same compression function, just using an externally allocated memory space to store compression state.
     Use LZ4_sizeofState() to know how much memory must be allocated,
     and allocate it on 8-bytes boundaries (using malloc() typically).
     Then, provide it as 'void* state' to compression function.
 
-    Use _safeExtState variant if LZ4_resetStream() was called on the state
-    buffer before being used for the first time (calls to this function leave
-    the state in a safe state, so zeroing is not required between calls).
-    Otherwise, using legacy _extState requires LZ4 to reinitialize the state
-    internally for every call.
+    Use the _noReset variant if LZ4_resetStream() was called on the state
+    buffer before being used for the first time (calls to both extState
+    functions leave the state in a safe state, so zeroing is not required
+    between calls). Otherwise, using the legacy _extState requires LZ4 to
+    reinitialize the state internally for every call.
 */
 LZ4LIB_API int LZ4_sizeofState(void);
-LZ4LIB_API int LZ4_compress_fast_safeExtState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 4cc2ef3..7acd7cf 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -722,7 +722,7 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize
     if (cdict) {
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
-        return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
+        return LZ4_compress_fast_extState_noReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
 }
 
diff --git a/tests/framebench.c b/tests/framebench.c
index a7a270b..fb4f38c 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -120,7 +120,7 @@ size_t compress_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_safeExtState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  oused = LZ4_compress_fast_extState_noReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
   obuf += oused;
 
   return obuf - p->obuf;
-- 
cgit v0.12


From 299f34909a2da7dc7096747e1c62cfe3994ec467 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 9 Mar 2018 12:05:31 -0500
Subject: Simpler Ternary Statements

---
 lib/lz4.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 94248be..6a680e6 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -582,9 +582,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
     const BYTE* const dictionary =
-        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictionary;
+        dictDirective == usingExtDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
-        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictSize;
+        dictDirective == usingExtDictCtx ? dictCtx->dictSize : cctx->dictSize;
 
     const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
     const BYTE* const dictEnd = dictionary + dictSize;
-- 
cgit v0.12


From 5149767a1b3caba4c17c0e79247c6092f7ae23ef Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 18:32:24 -0400
Subject: Add NULL Checks

---
 lib/lz4.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index 6a680e6..822d572 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -889,6 +889,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     int result;
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    if (ctxPtr == NULL) return 0;
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1101,6 +1102,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
 {
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    if (ctx == NULL) return 0;
 #else
     LZ4_stream_t ctxBody;
     LZ4_stream_t* ctx = &ctxBody;
@@ -1125,6 +1127,7 @@ LZ4_stream_t* LZ4_createStream(void)
     LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));
     LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal));    /* A compilation error here means LZ4_STREAMSIZE is not large enough */
     DEBUGLOG(4, "LZ4_createStream %p", lz4s);
+    if (lz4s == NULL) return NULL;
     LZ4_resetStream(lz4s);
     return lz4s;
 }
-- 
cgit v0.12


From b8e9c7785559b01f1f5ae0db6700e2673f4823db Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 18:46:54 -0400
Subject: Whitespace Fixes

---
 lib/lz4.c | 89 +++++++++++++++++++++++++++++++--------------------------------
 1 file changed, 44 insertions(+), 45 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 822d572..0209beb 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -529,35 +529,34 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
         const int inputSize,
         const tableType_t tableType,
         const dict_directive dictDirective) {
-  /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
-   * therefore safe to use no matter what mode we're in. Otherwise, we figure
-   * out if it's safe to leave as is or whether it needs to be reset.
-   */
-  if (cctx->tableType != unusedTable && (
-      cctx->tableType != tableType ||
-      (tableType == byU16 &&
-       cctx->currentOffset + inputSize >= 0xFFFFU) ||
-      (tableType == byU32 &&
-       cctx->currentOffset > 1 GB) ||
-      tableType == byPtr ||
-      inputSize >= 4 KB))
-  {
-      DEBUGLOG(4, "Resetting table in %p", cctx);
-      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 0;
-      cctx->tableType = unusedTable;
-  }
-  /* If the current offset is zero, we will never look in the external
-   * dictionary context, since there is no value a table entry can take that
-   * indicates a miss. In that case, we need to bump the offset to something
-   * non-zero.
-   */
-  if (dictDirective == usingExtDictCtx &&
-      tableType != byPtr &&
-      cctx->currentOffset == 0)
-  {
-      cctx->currentOffset = 1;
-  }
+    /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
+     * therefore safe to use no matter what mode we're in. Otherwise, we figure
+     * out if it's safe to leave as is or whether it needs to be reset.
+     */
+    if (cctx->tableType != unusedTable) {
+        if (cctx->tableType != tableType
+          || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU)
+          || (tableType == byU32 && cctx->currentOffset > 1 GB)
+          || tableType == byPtr
+          || inputSize >= 4 KB)
+        {
+            DEBUGLOG(4, "Resetting table in %p", cctx);
+            MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
+            cctx->currentOffset = 0;
+            cctx->tableType = unusedTable;
+        }
+    }
+    /* If the current offset is zero, we will never look in the external
+     * dictionary context, since there is no value a table entry can take that
+     * indicates a miss. In that case, we need to bump the offset to something
+     * non-zero.
+     */
+    if (dictDirective == usingExtDictCtx &&
+        tableType != byPtr &&
+        cctx->currentOffset == 0)
+    {
+        cctx->currentOffset = 1;
+    }
 }
 
 /** LZ4_compress_generic() :
@@ -793,12 +792,12 @@ _last_literals:
 
 _clean_up:
     if (dictDirective == usingExtDictCtx) {
-      /* Subsequent linked blocks can't use the dictionary. */
-      /* Instead, they use the block we just compressed. */
-      cctx->dictCtx = NULL;
-      cctx->dictSize = (U32)inputSize;
+        /* Subsequent linked blocks can't use the dictionary. */
+        /* Instead, they use the block we just compressed. */
+        cctx->dictCtx = NULL;
+        cctx->dictSize = (U32)inputSize;
     } else {
-      cctx->dictSize += (U32)inputSize;
+        cctx->dictSize += (U32)inputSize;
     }
     cctx->currentOffset += (U32)inputSize;
     cctx->tableType = tableType;
@@ -815,10 +814,10 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
     LZ4_resetStream((LZ4_stream_t*)state);
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, byU16, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {;
@@ -851,32 +850,32 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
             const tableType_t tableType = byU16;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              ctx->currentOffset += 64 KB;
+                ctx->currentOffset += 64 KB;
             }
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              ctx->currentOffset += 64 KB;
+                ctx->currentOffset += 64 KB;
             }
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
@@ -1282,9 +1281,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     LZ4_renormDictT(streamPtr, smallest);
 
     if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
     } else {
-      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
     }
 
     streamPtr->dictionary = (const BYTE*)source;
-- 
cgit v0.12


From 2be38a742917651359cdfc737b74bda8b50ece73 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 15:07:19 -0400
Subject: Rename Enums and Add Comment

---
 lib/lz4.c | 59 ++++++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 40 insertions(+), 19 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0209beb..f86d3ae 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -447,9 +447,32 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 *  Local Structures and types
 **************************************/
 typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
-typedef enum { unusedTable = 0, byPtr = 1, byU32 = 2, byU16 = 3 } tableType_t;
+typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t;
 
-typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingExtDictCtx } dict_directive;
+/**
+ * This enum distinguishes several different modes of accessing previous
+ * content in the stream.
+ *
+ * - noDict        : There is no preceding content.
+ * - withPrefix64k : Table entries up to ctx->dictSize before the current blob
+ *                   blob being compressed are valid and refer to the preceding
+ *                   content (of length ctx->dictSize), which is available
+ *                   contiguously preceding in memory the content currently
+ *                   being compressed.
+ * - usingExtDict  : Like withPrefix64k, but the preceding content is somewhere
+ *                   else in memory, starting at ctx->dictionary with length
+ *                   ctx->dictSize.
+ * - usingDictCtx  : Like usingExtDict, but everything concerning the preceding
+ *                   content is in a separate context, pointed to by
+ *                   ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table
+ *                   entries in the current context that refer to positions
+ *                   preceding the beginning of the current compression are
+ *                   ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx
+ *                   ->dictSize describe the location and size of the preceding
+ *                   content, and matches are found by looking in the ctx
+ *                   ->dictCtx->hashTable.
+ */
+typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive;
 typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
 
 typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
@@ -497,7 +520,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy
 {
     switch (tableType)
     {
-    case unusedTable: { /* illegal! */ assert(0); return; }
+    case clearedTable: { /* illegal! */ assert(0); return; }
     case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; }
     case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; }
     case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; }
@@ -533,7 +556,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * therefore safe to use no matter what mode we're in. Otherwise, we figure
      * out if it's safe to leave as is or whether it needs to be reset.
      */
-    if (cctx->tableType != unusedTable) {
+    if (cctx->tableType != clearedTable) {
         if (cctx->tableType != tableType
           || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU)
           || (tableType == byU32 && cctx->currentOffset > 1 GB)
@@ -543,7 +566,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
             DEBUGLOG(4, "Resetting table in %p", cctx);
             MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
             cctx->currentOffset = 0;
-            cctx->tableType = unusedTable;
+            cctx->tableType = clearedTable;
         }
     }
     /* If the current offset is zero, we will never look in the external
@@ -551,7 +574,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * indicates a miss. In that case, we need to bump the offset to something
      * non-zero.
      */
-    if (dictDirective == usingExtDictCtx &&
+    if (dictDirective == usingDictCtx &&
         tableType != byPtr &&
         cctx->currentOffset == 0)
     {
@@ -581,9 +604,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
     const BYTE* const dictionary =
-        dictDirective == usingExtDictCtx ? dictCtx->dictionary : cctx->dictionary;
+        dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
-        dictDirective == usingExtDictCtx ? dictCtx->dictSize : cctx->dictSize;
+        dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
 
     const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
     const BYTE* const dictEnd = dictionary + dictSize;
@@ -594,7 +617,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     /* the dictCtx currentOffset is indexed on the start of the dictionary,
      * while a dictionary in the current context precedes the currentOffset */
-    const BYTE* dictBase = dictDirective == usingExtDictCtx ?
+    const BYTE* dictBase = dictDirective == usingDictCtx ?
         (const BYTE*) source - dictCtx->currentOffset :
         (const BYTE*) source - dictSize - currentOffset;
     const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
@@ -640,10 +663,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 assert(ip < mflimitPlusOne);
 
                 match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
-                if (dictDirective == usingExtDictCtx) {
+                if (dictDirective == usingDictCtx) {
                     if (match < (const BYTE*)source) {
                         /* there was no match, try the dictionary */
-                        /* TODO: use precalc-ed hash? */
                         match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                         refDelta = dictDelta;
                         lowLimit = dictLowLimit;
@@ -696,7 +718,7 @@ _next_match:
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && lowLimit==dictionary) {
+            if ((dictDirective==usingExtDict || dictDirective==usingDictCtx) && lowLimit==dictionary) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
@@ -741,10 +763,9 @@ _next_match:
 
         /* Test next position */
         match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
-        if (dictDirective == usingExtDictCtx) {
+        if (dictDirective == usingDictCtx) {
             if (match < (const BYTE*)source) {
                 /* there was no match, try the dictionary */
-                /* TODO: use precalc-ed hash? */
                 match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                 refDelta = dictDelta;
                 lowLimit = dictLowLimit;
@@ -791,7 +812,7 @@ _last_literals:
     retval = (((char*)op)-dest);
 
 _clean_up:
-    if (dictDirective == usingExtDictCtx) {
+    if (dictDirective == usingDictCtx) {
         /* Subsequent linked blocks can't use the dictionary. */
         /* Instead, they use the block we just compressed. */
         cctx->dictCtx = NULL;
@@ -1135,7 +1156,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-    LZ4_stream->internal_donotuse.tableType = unusedTable;
+    LZ4_stream->internal_donotuse.tableType = clearedTable;
 }
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
@@ -1158,7 +1179,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
 
     if ((dict->initCheck)
-      || (dict->tableType != byU32 && dict->tableType != unusedTable)
+      || (dict->tableType != byU32 && dict->tableType != clearedTable)
       || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
         LZ4_resetStream(LZ4_dict);
 
@@ -1251,8 +1272,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDictCtx);
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
+                LZ4_prepareTable(streamPtr, inputSize, tableType, usingDictCtx);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
         } else {
             LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);
-- 
cgit v0.12


From c4aef7cd62cff0b71464dc58f40efc284bcb5a7b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 15:18:08 -0400
Subject: Restore checkTag Cleaning

---
 tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index 77d9e36..a351d35 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -113,7 +113,7 @@ clean:
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
         framebench$(EXT) \
-        fasttest$(EXT) datagen$(EXT)
+        fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
 
-- 
cgit v0.12


From 146e67653171b9b43322e2ab3d8be7b085b03bf7 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 15:42:03 -0400
Subject: Restore LZ4_sizeofStreamState, We Didn't Actually Need to Delete It

---
 lib/lz4.c | 3 +++
 lib/lz4.h | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index f86d3ae..48cdad1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1703,6 +1703,9 @@ They are only provided here for compatibility with older user programs.
 int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
 int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
 
+/* Obsolete Streaming functions */
+
+int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
 
 /* Obsolete streaming decompression functions */
 
diff --git a/lib/lz4.h b/lib/lz4.h
index ca9d552..e4a257b 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -477,6 +477,9 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
 LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
+/* Obsolete streaming functions; use new streaming interface whenever possible */
+LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
+
 /* Obsolete streaming decoding functions */
 LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
 LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
-- 
cgit v0.12


From 640db34e43481fdd7c69462ca8a84c9d964cdfd6 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 17:35:44 -0400
Subject: Another Allocation Fail Check

---
 lib/lz4frame.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 7acd7cf..d0c69d1 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -406,7 +406,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     size_t result;
 #if (LZ4F_HEAPMODE)
     LZ4F_cctx_t *cctxPtr;
-    LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
+    result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
+    if (LZ4F_isError(result)) return result;
 #else
     LZ4F_cctx_t cctx;
     LZ4_stream_t lz4ctx;
-- 
cgit v0.12


From 995756f218e4f144953ca4dfc8df37493e8f775b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 17:45:09 -0400
Subject: Split lz4CtxLevel into Two Fields

---
 lib/lz4frame.c | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d0c69d1..87e209f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -202,15 +202,8 @@ typedef struct LZ4F_cctx_s
     U64    totalInSize;
     XXH32_state_t xxh;
     void*  lz4CtxPtr;
-    /* lz4CtxLevel records both what size of memory has been allocated into the
-     * ctx pointer field and what it's being used as.
-     * - The low bit (& 1) value indicates whether the ctx is being used for
-     *   hc (1) or not (0).
-     * - The next bit (& 2) indicates whether the allocated memory is big
-     *   enough for a non-hc context.
-     * - The next bit (& 4) indicates whether the allocated memory is big
-     *   enough for an hc context. */
-    U32    lz4CtxLevel;
+    U16    lz4CtxAlloc; // sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx
+    U16    lz4CtxState; // in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx
 } LZ4F_cctx_t;
 
 
@@ -421,7 +414,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     {
         LZ4_resetStream(&lz4ctx);
         cctxPtr->lz4CtxPtr = &lz4ctx;
-        cctxPtr->lz4CtxLevel = 2;
+        cctxPtr->lz4CtxAlloc = 1;
+        cctxPtr->lz4CtxState = 1;
     }
 #endif
 
@@ -576,9 +570,8 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->prefs = *preferencesPtr;
 
     /* Ctx Management */
-    {   U32 const ctxSufficientAllocBits = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 2 : 6;
-        U32 const ctxTypeBit = cctxPtr->prefs.compressionLevel >= LZ4HC_CLEVEL_MIN;
-        if ((cctxPtr->lz4CtxLevel & ctxSufficientAllocBits) != ctxSufficientAllocBits) {
+    {   U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
+        if (cctxPtr->lz4CtxAlloc < ctxTypeID) {
             FREEMEM(cctxPtr->lz4CtxPtr);
             if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
@@ -586,17 +579,17 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
             }
             if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
-            cctxPtr->lz4CtxLevel = ctxSufficientAllocBits | ctxTypeBit;
-        } else if ((cctxPtr->lz4CtxLevel & 1) != ctxTypeBit) {
+            cctxPtr->lz4CtxAlloc = ctxTypeID;
+            cctxPtr->lz4CtxState = ctxTypeID;
+        } else if (cctxPtr->lz4CtxState != ctxTypeID) {
             /* otherwise, a sufficient buffer is allocated, but we need to
              * reset it to the correct context type */
             if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
                 LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
-                cctxPtr->lz4CtxLevel &= ~1;
             } else {
                 LZ4_resetStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
-                cctxPtr->lz4CtxLevel |= 1;
             }
+            cctxPtr->lz4CtxState = ctxTypeID;
         }
     }
 
-- 
cgit v0.12


From c852f20c39e877b877da49dea52dd4e36a5d6cb9 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 17:47:34 -0400
Subject: Switch ALLOC() to ALLOC_AND_ZERO() to Paper Over Existing
 Uninitialized Read

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 87e209f..b91cb7c 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -605,7 +605,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         if (cctxPtr->maxBufferSize < requiredBuffSize) {
             cctxPtr->maxBufferSize = 0;
             FREEMEM(cctxPtr->tmpBuff);
-            cctxPtr->tmpBuff = (BYTE*)ALLOC(requiredBuffSize);
+            cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize);
             if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             cctxPtr->maxBufferSize = requiredBuffSize;
     }   }
-- 
cgit v0.12


From 66b6fbfe6fa9d7e8d4686e6525883ff4aef31e02 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 14 Mar 2018 15:51:59 -0400
Subject: Restore the Other Old Streaming Functions in a Degraded Fashion

---
 lib/lz4.c | 19 +++++++++++++++++++
 lib/lz4.h | 11 +++++++++--
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 48cdad1..37782f5 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1707,6 +1707,25 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize,
 
 int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
 
+int LZ4_resetStreamState(void* state, char* inputBuffer)
+{
+    (void)inputBuffer;
+    LZ4_resetStream((LZ4_stream_t*)state);
+    return 0;
+}
+
+void* LZ4_create (char* inputBuffer)
+{
+    (void)inputBuffer;
+    return LZ4_createStream();
+}
+
+char* LZ4_slideInputBuffer (void* state)
+{
+    // avoid const char * -> char * conversion warning
+    return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary;
+}
+
 /* Obsolete streaming decompression functions */
 
 int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
diff --git a/lib/lz4.h b/lib/lz4.h
index e4a257b..6cb6589 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -477,8 +477,15 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
 LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
-/* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
+/* Broken, obsolete streaming functions; do not use!
+ *
+ * These functions depended on data that is no longer tracked in the state. They
+ * are therefore broken--they don't retain any history across compressions.
+ */
+LZ4_DEPRECATED("Broken!!! Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
+LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
+LZ4_DEPRECATED("Broken!!! Use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4_DEPRECATED("Broken!!! Use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
 
 /* Obsolete streaming decoding functions */
 LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
-- 
cgit v0.12


From b0a18896fe53cf2b69142c092f955e57b2b0d8df Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 14 Mar 2018 15:58:38 -0400
Subject: Move LZ4_compress_fast_extState_noReset Declaration to Unstable
 Section

---
 lib/lz4.h | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 6cb6589..d0ec204 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -179,21 +179,13 @@ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int d
 
 
 /*!
-LZ4_compress_fast_extState_noReset() :
 LZ4_compress_fast_extState() :
     Same compression function, just using an externally allocated memory space to store compression state.
     Use LZ4_sizeofState() to know how much memory must be allocated,
     and allocate it on 8-bytes boundaries (using malloc() typically).
     Then, provide it as 'void* state' to compression function.
-
-    Use the _noReset variant if LZ4_resetStream() was called on the state
-    buffer before being used for the first time (calls to both extState
-    functions leave the state in a safe state, so zeroing is not required
-    between calls). Otherwise, using the legacy _extState requires LZ4 to
-    reinitialize the state internally for every call.
 */
 LZ4LIB_API int LZ4_sizeofState(void);
-LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
@@ -351,6 +343,29 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 /*^**********************************************
  * !!!!!!   STATIC LINKING ONLY   !!!!!!
  ***********************************************/
+
+/*-************************************
+ *  Unstable declarations
+ **************************************
+ * Declarations in this section should be considered unstable.
+ * Use at your own peril, etc., etc.
+ * They may be removed in the future.
+ * Their signatures may change.
+ **************************************/
+
+/*!
+LZ4_compress_fast_extState_noReset() :
+    A variant of LZ4_compress_fast_extState().
+
+    Use the _noReset variant if LZ4_resetStream() was called on the state
+    buffer before being used for the first time (calls to both extState
+    functions leave the state in a safe state, so zeroing is not required
+    between calls). Otherwise, using the plain _extState requires LZ4 to
+    reinitialize the state internally for every call.
+*/
+LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+
+
 /*-************************************
  *  Private definitions
  **************************************
-- 
cgit v0.12


From 70f14823a46719e81e808d9ed9df90f478bcfd3f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 31 Jan 2018 18:11:37 -0500
Subject: Remove Framebench Tool

---
 tests/Makefile     |   6 +-
 tests/framebench.c | 295 -----------------------------------------------------
 2 files changed, 1 insertion(+), 300 deletions(-)
 delete mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index a351d35..5954370 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -62,7 +62,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen framebench
+all: fullbench fuzzer frametest datagen
 
 all32: CFLAGS+=-m32
 all32: all
@@ -98,9 +98,6 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
-	$(CC) $(FLAGS) $^ -o $@$(EXT)
-
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -112,7 +109,6 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
-        framebench$(EXT) \
         fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
diff --git a/tests/framebench.c b/tests/framebench.c
deleted file mode 100644
index fb4f38c..0000000
--- a/tests/framebench.c
+++ /dev/null
@@ -1,295 +0,0 @@
-#include <assert.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <time.h>
-#include <unistd.h>
-
-#include "lz4.h"
-#include "lz4frame.h"
-#include "lz4frame_static.h"
-
-#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
-
-typedef struct {
-  size_t iter;
-  LZ4_stream_t *ctx;
-  LZ4F_cctx *cctx;
-  char *obuf;
-  size_t osize;
-  const char* ibuf;
-  size_t isize;
-  size_t num_ibuf;
-  const LZ4F_CDict* cdict;
-  LZ4F_preferences_t* prefs;
-  const LZ4F_compressOptions_t* options;
-} bench_params_t;
-
-size_t compress_frame(bench_params_t *p) {
-  size_t iter = p->iter;
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressFrame_usingCDict(
-    cctx,
-    obuf,
-    osize,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
-    isize,
-    cdict,
-    prefs);
-  LZ4F_CHECK(oused);
-
-  return oused;
-}
-
-size_t compress_begin(bench_params_t *p) {
-  size_t iter = p->iter;
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-  const LZ4F_compressOptions_t* options = p->options;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressUpdate(
-    cctx,
-    obuf,
-    oend - obuf,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
-    isize,
-    options);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
-  LZ4F_CHECK(oused);
-
-  return obuf - p->obuf;
-}
-
-size_t compress_default(bench_params_t *p) {
-  size_t iter = p->iter;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_extState(bench_params_t *p) {
-  size_t iter = p->iter;
-  LZ4_stream_t *ctx = p->ctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_fast_extState_noReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-uint64_t bench(
-    size_t (*fun)(bench_params_t *),
-    uint64_t repetitions,
-    bench_params_t *params,
-    size_t *osizePtr
-) {
-  struct timespec start, end;
-  size_t i, osize = 0;
-
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
-
-  for (i = 0; i < repetitions; i++) {
-    size_t o;
-    params->iter = i;
-    o = fun(params);
-    if (!o) return 0;
-    osize += o;
-  }
-
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
-
-  *osizePtr = osize / repetitions;
-  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
-}
-
-int main(int argc, char *argv[]) {
-
-
-  struct stat st;
-  size_t bytes_read;
-
-  const char *dict_fn;
-  size_t dict_size;
-  char *dict_buf;
-  FILE *dict_file;
-
-  const char *in_fn;
-  size_t in_size;
-  size_t num_in_buf;
-  size_t cur_in_buf;
-  char *in_buf;
-  FILE *in_file;
-
-  size_t out_size;
-  char *out_buf;
-
-  LZ4_stream_t *ctx;
-  LZ4F_cctx *cctx;
-  LZ4F_CDict *cdict;
-  LZ4F_preferences_t prefs;
-  LZ4F_compressOptions_t options;
-
-  size_t out_used;
-
-  uint64_t time_taken;
-  uint64_t repetitions;
-
-  bench_params_t params;
-
-  if (argc != 3) return 1;
-  dict_fn = argv[1];
-  in_fn = argv[2];
-
-  if (stat(dict_fn, &st)) return 1;
-  dict_size = st.st_size;
-  dict_buf = (char *)malloc(dict_size);
-  if (!dict_buf) return 1;
-  dict_file = fopen(dict_fn, "r");
-  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
-  if (bytes_read != dict_size) return 1;
-
-  if (stat(in_fn, &st)) return 1;
-  in_size = st.st_size;
-  num_in_buf = 256 * 1024 * 1024 / in_size;
-  if (num_in_buf == 0) {
-    num_in_buf = 1;
-  }
-
-  in_buf = (char *)malloc(in_size * num_in_buf);
-  if (!in_buf) return 1;
-  in_file = fopen(in_fn, "r");
-  bytes_read = fread(in_buf, 1, in_size, in_file);
-  if (bytes_read != in_size) return 1;
-
-  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
-    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
-  }
-
-  if (in_size <= 1024) {
-    repetitions = 100000;
-  } else
-  if (in_size <= 16384) {
-    repetitions = 10000;
-  } else
-  if (in_size <= 131072) {
-    repetitions = 1000;
-  } else
-  if (in_size <= 1048576) {
-    repetitions = 100;
-  } else {
-    repetitions = 50;
-  }
-
-  memset(&prefs, 0, sizeof(prefs));
-  prefs.autoFlush = 1;
-  if (in_size < 64 * 1024)
-    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
-  prefs.frameInfo.contentSize = in_size;
-
-  memset(&options, 0, sizeof(options));
-  options.stableSrc = 1;
-
-  out_size = LZ4F_compressFrameBound(in_size, &prefs);
-  out_buf = (char *)malloc(out_size);
-  if (!out_buf) return 1;
-
-  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
-  if (cctx == NULL) return 1;
-
-  ctx = LZ4_createStream();
-  if (ctx == NULL) return 1;
-
-  cdict = LZ4F_createCDict(dict_buf, dict_size);
-  if (!cdict) return 1;
-
-  fprintf(stderr, "dict  size: %zd\n", dict_size);
-  fprintf(stderr, "input size: %zd\n", in_size);
-
-  params.ctx = ctx;
-  params.cctx = cctx;
-  params.obuf = out_buf;
-  params.osize = out_size;
-  params.ibuf = in_buf;
-  params.isize = in_size;
-  params.num_ibuf = num_in_buf;
-  params.cdict = NULL;
-  params.prefs = &prefs;
-  params.options = &options;
-
-  time_taken = bench(compress_default, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_extState, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  params.cdict = cdict;
-
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  return 0;
-}
\ No newline at end of file
-- 
cgit v0.12


From d6711a7cffe3b9f7104ea48c193be7d7fc9e4d69 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 19 Mar 2018 16:18:10 -0700
Subject: minor man fix on clevels

---
 programs/lz4.1.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index b7b8570..a5168e9 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -117,9 +117,9 @@ only the latest one will be applied.
 ### Operation modifiers
 
 * `-#`:
-  Compression level, with # being any value from 1 to 16.
+  Compression level, with # being any value from 1 to 12.
   Higher values trade compression speed for compression ratio.
-  Values above 16 are considered the same as 16.
+  Values above 12 are considered the same as 12.
   Recommended values are 1 for fast compression (default),
   and 9 for high compression.
   Speed/compression trade-off will vary depending on data to compress.
@@ -206,7 +206,7 @@ only the latest one will be applied.
   Benchmark multiple compression levels, from b# to e# (included)
 
 * `-i#`:
-  Minimum evaluation in seconds \[1-9\] (default : 3)
+  Minimum evaluation time in seconds \[1-9\] (default : 3)
 
 
 BUGS
-- 
cgit v0.12


From 1faa7e2698e73864aa20729caba8071148af5b4c Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 19 Mar 2018 17:19:25 -0700
Subject: bench: introduced hidden command -S

to benchmark multiple files with separate results
---
 programs/bench.c  | 34 +++++++++++++++++++++++++++-------
 programs/bench.h  |  5 +++--
 programs/lz4cli.c | 10 +++++++---
 3 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 002eac9..770191c 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -119,21 +119,21 @@ static clock_t g_time = 0;
 static U32 g_nbSeconds = NBSECONDS;
 static size_t g_blockSize = 0;
 int g_additionalParam = 0;
+int g_benchSeparately = 0;
 
 void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 
 void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
 
-void BMK_SetNbSeconds(unsigned nbSeconds)
+void BMK_setNbSeconds(unsigned nbSeconds)
 {
     g_nbSeconds = nbSeconds;
     DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
 }
 
-void BMK_SetBlockSize(size_t blockSize)
-{
-    g_blockSize = blockSize;
-}
+void BMK_setBlockSize(size_t blockSize) { g_blockSize = blockSize; }
+
+void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); }
 
 
 /* ********************************************************
@@ -515,6 +515,22 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
 }
 
 
+int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
+                   int cLevel, int cLevelLast)
+{
+    unsigned fileNb;
+    if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
+    if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
+    if (cLevelLast < cLevel) cLevelLast = cLevel;
+    if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
+
+    for (fileNb=0; fileNb<nbFiles; fileNb++)
+        BMK_benchFileTable(fileNamesTable+fileNb, 1, cLevel, cLevelLast);
+
+    return 0;
+}
+
+
 int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
                    int cLevel, int cLevelLast)
 {
@@ -527,7 +543,11 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
 
     if (nbFiles == 0)
         BMK_syntheticTest(cLevel, cLevelLast, compressibility);
-    else
-        BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast);
+    else {
+        if (g_benchSeparately)
+            BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast);
+        else
+            BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast);
+    }
     return 0;
 }
diff --git a/programs/bench.h b/programs/bench.h
index 15def93..bb67bee 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -29,9 +29,10 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
                    int cLevel, int cLevelLast);
 
 /* Set Parameters */
-void BMK_SetNbSeconds(unsigned nbLoops);
-void BMK_SetBlockSize(size_t blockSize);
+void BMK_setNbSeconds(unsigned nbLoops);
+void BMK_setBlockSize(size_t blockSize);
 void BMK_setAdditionalParam(int additionalParam);
 void BMK_setNotificationLevel(unsigned level);
+void BMK_setBenchSeparately(int separate);
 
 #endif   /* BENCH_H_125623623633 */
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 362ba49..42392eb 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -458,11 +458,11 @@ int main(int argc, const char** argv)
                                 if (B < 4) badusage(exeName);
                                 if (B <= 7) {
                                     blockSize = LZ4IO_setBlockSizeID(B);
-                                    BMK_SetBlockSize(blockSize);
+                                    BMK_setBlockSize(blockSize);
                                     DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
                                 } else {
                                     if (B < 32) badusage(exeName);
-                                    BMK_SetBlockSize(B);
+                                    BMK_setBlockSize(B);
                                     if (B >= 1024) {
                                         DISPLAYLEVEL(2, "bench: using blocks of size %u KB \n", (U32)(B>>10));
                                     } else {
@@ -480,6 +480,10 @@ int main(int argc, const char** argv)
                 case 'b': mode = om_bench; multiple_inputs=1;
                     break;
 
+                    /* hidden command : benchmark files, but do not fuse result */
+                case 'S': BMK_setBenchSeparately(1);
+                    break;
+
 #ifdef UTIL_HAS_CREATEFILELIST
                     /* recursive */
                 case 'r': recursive=1;
@@ -496,7 +500,7 @@ int main(int argc, const char** argv)
                         iters = readU32FromChar(&argument);
                         argument--;
                         BMK_setNotificationLevel(displayLevel);
-                        BMK_SetNbSeconds(iters);   /* notification if displayLevel >= 3 */
+                        BMK_setNbSeconds(iters);   /* notification if displayLevel >= 3 */
                     }
                     break;
 
-- 
cgit v0.12


From 863e24892d853ea8100c9224c8f3a0c917e055c7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 21 Mar 2018 07:07:24 -0700
Subject: fix comment style

---
 lib/lz4.c      | 2 +-
 lib/lz4frame.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 37782f5..0fdbe5e 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1722,7 +1722,7 @@ void* LZ4_create (char* inputBuffer)
 
 char* LZ4_slideInputBuffer (void* state)
 {
-    // avoid const char * -> char * conversion warning
+    /* avoid const char * -> char * conversion warning */
     return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary;
 }
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index b91cb7c..a080fa6 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -202,8 +202,8 @@ typedef struct LZ4F_cctx_s
     U64    totalInSize;
     XXH32_state_t xxh;
     void*  lz4CtxPtr;
-    U16    lz4CtxAlloc; // sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx
-    U16    lz4CtxState; // in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx
+    U16    lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
+    U16    lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
 } LZ4F_cctx_t;
 
 
-- 
cgit v0.12


From c3f0ed28ffa66fd7e28ec3b6dbbe95eb0974bfef Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Wed, 21 Mar 2018 07:14:13 -0700
Subject: added low address fuzzer tests

---
 tests/Makefile |   3 +-
 tests/fuzzer.c | 113 +++++++++++++++++++++++++++++++--------------------------
 2 files changed, 63 insertions(+), 53 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index ddc0d2e..34b8b24 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -35,7 +35,8 @@ PRGDIR  := ../programs
 TESTDIR := versionsTest
 PYTHON  ?= python3
 
-DEBUGFLAGS = -g -DLZ4_DEBUG=1
+DEBUGLEVEL?= 1
+DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
 CFLAGS  ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make
 CFLAGS  += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
            -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 9415e94..2b9b926 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -34,9 +34,13 @@
 
 #define LZ4_DISABLE_DEPRECATE_WARNINGS
 
+
 /*-************************************
 *  Dependencies
 **************************************/
+#ifdef __unix__   /* must be included before platform.h for MAP_ANONYMOUS */
+#  include <sys/mman.h>   /* mmap */
+#endif
 #include "platform.h"   /* _CRT_SECURE_NO_WARNINGS */
 #include "util.h"       /* U32 */
 #include <stdlib.h>
@@ -242,8 +246,6 @@ _overflowError:
 
 #ifdef __unix__   /* is expected to be triggered on linux+gcc */
 
-#include <sys/mman.h>   /* mmap */
-
 static void* FUZ_createLowAddr(size_t size)
 {
     void* const lowBuff = mmap((void*)(0x1000), size,
@@ -276,6 +278,7 @@ static void FUZ_freeLowAddr(void* buffer, size_t size)
 
 #endif
 
+
 /*! FUZ_findDiff() :
 *   find the first different byte between buff1 and buff2.
 *   presumes buff1 != buff2.
@@ -316,10 +319,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
 #   define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
                                                 printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
-#   define FUZ_DISPLAYTEST          { testNb++; g_displayLevel>=4 ? printf("%2u\b\b", testNb), fflush(stdout) : 0; }
+#   define FUZ_DISPLAYTEST(...) {                 \
+                testNb++;                         \
+                if (g_displayLevel>=4) {          \
+                    printf("\r%4u - %2u ", seed, testNb);  \
+                    printf(" " __VA_ARGS__);      \
+                    printf("   ");                \
+                    fflush(stdout);               \
+            }   }
 
 
     /* init */
+    DISPLAYLEVEL(2, " g_displayLevel = %u \n", g_displayLevel);
     if(!CNBuffer || !compressedBuffer || !decodedBuffer) {
         DISPLAY("Not enough memory to start fuzzer tests");
         goto _output_error;
@@ -362,7 +373,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
 
         /* Test compression destSize */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_destSize()");
         {   int srcSize = blockSize;
             int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
             char endCheck = FUZ_rand(&randState) & 255;
@@ -377,7 +388,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
                 U32 const crcBase = XXH32(block, srcSize, 0);
                 char const canary = FUZ_rand(&randState) & 255;
                 FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed");
-                FUZ_DISPLAYTEST;
+                FUZ_DISPLAYTEST();
                 compressedSize = ret;
                 decodedBuffer[srcSize] = canary;
                 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
@@ -393,7 +404,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Test compression HC destSize */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()");
         {   int srcSize = blockSize;
             int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
             char const endCheck = FUZ_rand(&randState) & 255;
@@ -407,14 +418,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
             FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !");
             FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !");
-            DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
-                            compressionLevel, ret, targetSize, srcSize, blockSize);
             if (targetSize>0) {
                 /* check correctness */
                 U32 const crcBase = XXH32(block, srcSize, 0);
                 char const canary = FUZ_rand(&randState) & 255;
                 FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed");
-                FUZ_DISPLAYTEST;
+                FUZ_DISPLAYTEST();
                 compressedSize = ret;
                 decodedBuffer[srcSize] = canary;
                 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
@@ -430,31 +439,31 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Test compression HC */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC()");
         ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed");
         HCcompressedSize = ret;
 
         /* Test compression HC using external state */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()");
         ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed");
 
         /* Test compression using external state */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()");
         ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
 
         /* Test compression */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_default()");
         ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed");
         compressedSize = ret;
 
         /* Decompression tests */
 
-        /* Test decoding with output size being exactly what's necessary => must work */
-        FUZ_DISPLAYTEST;
+        /* Test decoding with output size exactly correct => must work */
+        FUZ_DISPLAYTEST("LZ4_decompress_fast() with exact output buffer");
         ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
         FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
@@ -462,19 +471,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
 
         /* Test decoding with one byte missing => must fail */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
 
         /* Test decoding with one byte too much => must fail */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
 
         /* Test decoding with output size exactly what's necessary => must work */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
@@ -484,7 +493,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
 
         // Test decoding with more than enough output size => must work
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         decodedBuffer[blockSize+1] = 0;
         ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
@@ -496,14 +505,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
 
         // Test decoding with output size being one byte too short => must fail
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
 
         // Test decoding with output size being 10 bytes too short => must fail
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         if (blockSize>10) {
             decodedBuffer[blockSize-10] = 0;
             ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
@@ -512,51 +521,51 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
 
         // Test decoding with input size being one byte too short => must fail
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
 
         // Test decoding with input size being one byte too large => must fail
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
 
         // Test partial decoding with target output size being max/2 => must work
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize);
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
 
         // Test partial decoding with target output size being just below max => must work
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize);
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
 
         /* Test Compression with limited output size */
 
         /* Test compression with output size being exactly what's necessary (should work) */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer just the right size");
         ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed despite sufficient space");
 
         /* Test compression with output size being exactly what's necessary and external state (should work) */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_extState() with output buffer just the right size");
         ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed despite sufficient space");
 
         /* Test HC compression with output size being exactly what's necessary (should work) */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer just the right size");
         ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed despite sufficient space");
 
         /* Test HC compression with output size being exactly what's necessary (should work) */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC() with output buffer just the right size");
         ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed despite sufficient space");
 
         /* Test compression with missing bytes into output buffer => must fail */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer a bit too short");
         {   int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1;
             if (missingBytes >= compressedSize) missingBytes = compressedSize-1;
             missingBytes += !missingBytes;   /* avoid special case missingBytes==0 */
@@ -567,7 +576,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
 
         /* Test HC compression with missing bytes into output buffer => must fail */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer a bit too short");
         {   int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1;
             if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1;
             missingBytes += !missingBytes;   /* avoid special case missingBytes==0 */
@@ -583,7 +592,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /*-******************/
 
         /* Compress using dictionary */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary of size %i", dictSize);
         {   LZ4_stream_t LZ4_stream;
             LZ4_resetStream(&LZ4_stream);
             LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1);   /* Just to fill hash tables */
@@ -592,7 +601,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
 
         /* Decompress with dictionary as prefix */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as prefix");
         memcpy(decodedBuffer, dict, dictSize);
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
@@ -605,33 +614,33 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()");
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
         crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
 
         /* Compress using External dictionary */
-        FUZ_DISPLAYTEST;
-        dict -= (FUZ_rand(&randState) & 0xF) + 1;   /* Separation, so it is an ExtDict */
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary");
+        dict -= (FUZ_rand(&randState) & 0xF) + 1;   /* create space, so now dictionary is an ExtDict */
         if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
         LZ4_loadDict(&LZ4dict, dict, dictSize);
         blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
         FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary but with an output buffer too short by one byte");
         LZ4_loadDict(&LZ4dict, dict, dictSize);
         ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
         FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()");
         LZ4_loadDict(&LZ4dict, dict, dictSize);
         ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
         FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
 
         /* Decompress with dictionary as external */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
@@ -640,7 +649,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
@@ -648,19 +657,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         crcCheck = XXH32(decodedBuffer, blockSize, 0);
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         {   U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
             if ((U32)blockSize > missingBytes) {
                 decodedBuffer[blockSize-missingBytes] = 0;
@@ -670,7 +679,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Compress HC using External dictionary */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         dict -= (FUZ_rand(&randState) & 7);    /* even bigger separation */
         if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
         LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
@@ -679,18 +688,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
         FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
         ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
         FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
         ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize);
         FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
@@ -701,10 +710,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
 
         /* Compress HC continue destSize */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         {   int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5;
             int consumedSize = blockSize;
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
             LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
             blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
@@ -713,7 +722,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow");
             FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow");
 
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             decodedBuffer[consumedSize] = 0;
             ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize);
             FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
@@ -1118,13 +1127,13 @@ int main(int argc, const char** argv)
                     return FUZ_usage(programName);
 
                 case 'v':   /* verbose mode */
-                    argument++;
                     g_displayLevel++;
+                    argument++;
                     break;
 
                 case 'p':   /* pause at the end */
-                    argument++;
                     use_pause=1;
+                    argument++;
                     break;
 
                 case 'i':
-- 
cgit v0.12


From 7b4c448571b678978bf8fc77e7ba89759086b672 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 21 Mar 2018 07:19:48 -0700
Subject: added c90 test to c_standards

to catch `//` comments
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index ff84d6d..3a95808 100644
--- a/Makefile
+++ b/Makefile
@@ -173,7 +173,7 @@ gpptest gpptest32: clean
 	CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
 
 c_standards: clean
-	# note : lz4 is not C90 compatible, because it requires long long support
+	CFLAGS="-std=c90   -Werror" $(MAKE) clean allmost
 	CFLAGS="-std=gnu90 -Werror" $(MAKE) clean allmost
 	CFLAGS="-std=c99   -Werror" $(MAKE) clean allmost
 	CFLAGS="-std=gnu99 -Werror" $(MAKE) clean allmost
-- 
cgit v0.12


From ebdcbc359baa5a836a3c9e647f083fe092456dd4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 11:28:51 -0400
Subject: Add Dependency to Fix Parallel `make test` Runs

When run with `-jN`, the `rm tmp*` can run in the middle of the `test-lz4-dict`
job, which will then fail, finding its files to have been axed. This adds a
dependency between the two.
---
 tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index 5954370..77f5d02 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -327,7 +327,7 @@ test-lz4-opt-parser: lz4 datagen
 
 test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple \
                       test-lz4-frame-concatenation test-lz4-testmode \
-                      test-lz4-contentSize
+                      test-lz4-contentSize test-lz4-dict
 	@$(RM) tmp*
 
 test-lz4: lz4 datagen test-lz4-essentials test-lz4-opt-parser \
-- 
cgit v0.12


From a3a9b80dffc4d2c8e2496b851f78b78f0948c4bf Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 11:39:41 -0400
Subject: Better Describe Functionality of Obsolete Streaming Functions

---
 lib/lz4.h | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index d0ec204..80f040f 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -492,15 +492,19 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
 LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
-/* Broken, obsolete streaming functions; do not use!
+/* Obsolete streaming functions; degraded functionality; do not use!
  *
- * These functions depended on data that is no longer tracked in the state. They
- * are therefore broken--they don't retain any history across compressions.
+ * In order to perform streaming compression, these functions depended on data
+ * that is no longer tracked in the state. They have been preserved as well as
+ * possible: using them will still produce a correct output. However, they don't
+ * actually retain any history between compression calls. The compression ratio
+ * achieved will therefore be no better than compressing each chunk
+ * independently.
  */
-LZ4_DEPRECATED("Broken!!! Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
+LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
 LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
-LZ4_DEPRECATED("Broken!!! Use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4_DEPRECATED("Broken!!! Use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
+LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
 
 /* Obsolete streaming decoding functions */
 LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
-- 
cgit v0.12


From 126f18d3e09d92c06d06d33d9cb7a1ec51962525 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 11:48:35 -0400
Subject: Also Fix a Comment

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 37782f5..0fdbe5e 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1722,7 +1722,7 @@ void* LZ4_create (char* inputBuffer)
 
 char* LZ4_slideInputBuffer (void* state)
 {
-    // avoid const char * -> char * conversion warning
+    /* avoid const char * -> char * conversion warning */
     return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary;
 }
 
-- 
cgit v0.12


From 606afdb16428347d0bf216b2c614d962018b3fc3 Mon Sep 17 00:00:00 2001
From: Kenji Chan <kenjichanhkg@gmail.com>
Date: Mon, 2 Apr 2018 10:52:45 +0800
Subject: added vs2017 projects

---
 visual/VS2017/datagen/datagen.vcxproj             | 173 ++++++++++++++++++++
 visual/VS2017/frametest/frametest.vcxproj         | 180 +++++++++++++++++++++
 visual/VS2017/fullbench-dll/fullbench-dll.vcxproj | 184 ++++++++++++++++++++++
 visual/VS2017/fullbench/fullbench.vcxproj         | 180 +++++++++++++++++++++
 visual/VS2017/fuzzer/fuzzer.vcxproj               | 177 +++++++++++++++++++++
 visual/VS2017/liblz4-dll/liblz4-dll.rc            |  51 ++++++
 visual/VS2017/liblz4-dll/liblz4-dll.vcxproj       | 183 +++++++++++++++++++++
 visual/VS2017/liblz4/liblz4.vcxproj               | 179 +++++++++++++++++++++
 visual/VS2017/lz4.sln                             |  98 ++++++++++++
 9 files changed, 1405 insertions(+)
 create mode 100644 visual/VS2017/datagen/datagen.vcxproj
 create mode 100644 visual/VS2017/frametest/frametest.vcxproj
 create mode 100644 visual/VS2017/fullbench-dll/fullbench-dll.vcxproj
 create mode 100644 visual/VS2017/fullbench/fullbench.vcxproj
 create mode 100644 visual/VS2017/fuzzer/fuzzer.vcxproj
 create mode 100644 visual/VS2017/liblz4-dll/liblz4-dll.rc
 create mode 100644 visual/VS2017/liblz4-dll/liblz4-dll.vcxproj
 create mode 100644 visual/VS2017/liblz4/liblz4.vcxproj
 create mode 100644 visual/VS2017/lz4.sln

diff --git a/visual/VS2017/datagen/datagen.vcxproj b/visual/VS2017/datagen/datagen.vcxproj
new file mode 100644
index 0000000..30e159e
--- /dev/null
+++ b/visual/VS2017/datagen/datagen.vcxproj
@@ -0,0 +1,173 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{D745AE2F-596A-403A-9B91-81A8C6779243}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>datagen</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\programs\datagen.c" />
+    <ClCompile Include="..\..\..\tests\datagencli.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\programs\datagen.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/frametest/frametest.vcxproj b/visual/VS2017/frametest/frametest.vcxproj
new file mode 100644
index 0000000..a3a403d
--- /dev/null
+++ b/visual/VS2017/frametest/frametest.vcxproj
@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>frametest</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4frame.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+    <ClCompile Include="..\..\..\tests\frametest.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj
new file mode 100644
index 0000000..d54a8d7
--- /dev/null
+++ b/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj
@@ -0,0 +1,184 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{13992FD2-077E-4954-B065-A428198201A9}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>fullbench-dll</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+    <ClCompile Include="..\..\..\tests\fullbench.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/fullbench/fullbench.vcxproj b/visual/VS2017/fullbench/fullbench.vcxproj
new file mode 100644
index 0000000..54c9743
--- /dev/null
+++ b/visual/VS2017/fullbench/fullbench.vcxproj
@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>fullbench</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4frame.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+    <ClCompile Include="..\..\..\tests\fullbench.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/fuzzer/fuzzer.vcxproj b/visual/VS2017/fuzzer/fuzzer.vcxproj
new file mode 100644
index 0000000..aa6fe42
--- /dev/null
+++ b/visual/VS2017/fuzzer/fuzzer.vcxproj
@@ -0,0 +1,177 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{18B9F1A7-9C66-4352-898B-30804DADE0FD}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>fuzzer</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+    <ClCompile Include="..\..\..\tests\fuzzer.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/liblz4-dll/liblz4-dll.rc b/visual/VS2017/liblz4-dll/liblz4-dll.rc
new file mode 100644
index 0000000..b1871fe
--- /dev/null
+++ b/visual/VS2017/liblz4-dll/liblz4-dll.rc
@@ -0,0 +1,51 @@
+// Microsoft Visual C++ generated resource script.
+//
+
+#include "lz4.h" /* LZ4_VERSION_STRING */
+#define APSTUDIO_READONLY_SYMBOLS
+#include "verrsrc.h"
+#undef APSTUDIO_READONLY_SYMBOLS
+
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+LANGUAGE 9, 1
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+VS_VERSION_INFO     VERSIONINFO
+  FILEVERSION       LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
+  PRODUCTVERSION    LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
+ FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
+#ifdef _DEBUG
+ FILEFLAGS VS_FF_DEBUG
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS VOS_NT_WINDOWS32
+ FILETYPE VFT_DLL
+ FILESUBTYPE VFT2_UNKNOWN
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904B0"
+        BEGIN
+            VALUE "CompanyName", "Yann Collet"
+            VALUE "FileDescription", "Extremely fast compression"
+            VALUE "FileVersion", LZ4_VERSION_STRING
+            VALUE "InternalName", "lz4.dll"
+            VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+            VALUE "OriginalFilename", "lz4.dll"
+            VALUE "ProductName", "LZ4"
+            VALUE "ProductVersion", LZ4_VERSION_STRING
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x0409, 1200
+    END
+END
+
+#endif
diff --git a/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj
new file mode 100644
index 0000000..8e7ee3b
--- /dev/null
+++ b/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj
@@ -0,0 +1,183 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{9800039D-4AAA-43A4-BB78-FEF6F4836927}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>liblz4-dll</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+    <ProjectName>liblz4-dll</ProjectName>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <TargetName>liblz4</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <TargetName>liblz4</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetName>liblz4</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetName>liblz4</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4frame.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ResourceCompile Include="liblz4-dll.rc" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/liblz4/liblz4.vcxproj b/visual/VS2017/liblz4/liblz4.vcxproj
new file mode 100644
index 0000000..948f7db
--- /dev/null
+++ b/visual/VS2017/liblz4/liblz4.vcxproj
@@ -0,0 +1,179 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>liblz4</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <TargetName>liblz4_static</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <TargetName>liblz4_static</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetName>liblz4_static</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetName>liblz4_static</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4frame.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/lz4.sln b/visual/VS2017/lz4.sln
new file mode 100644
index 0000000..78f223b
--- /dev/null
+++ b/visual/VS2017/lz4.sln
@@ -0,0 +1,98 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Express 2012 for Windows Desktop
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{E30329AC-0057-4FE0-8FDA-7F650D398C4C}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4-dll", "liblz4-dll\liblz4-dll.vcxproj", "{9800039D-4AAA-43A4-BB78-FEF6F4836927}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4", "liblz4\liblz4.vcxproj", "{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "frametest", "frametest\frametest.vcxproj", "{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{D745AE2F-596A-403A-9B91-81A8C6779243}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll\fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}"
+	ProjectSection(ProjectDependencies) = postProject
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927} = {9800039D-4AAA-43A4-BB78-FEF6F4836927}
+	EndProjectSection
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.ActiveCfg = Debug|Win32
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.Build.0 = Debug|Win32
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|x64.ActiveCfg = Debug|x64
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|x64.Build.0 = Debug|x64
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|Win32.ActiveCfg = Release|Win32
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|Win32.Build.0 = Release|Win32
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|x64.ActiveCfg = Release|x64
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|x64.Build.0 = Release|x64
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.ActiveCfg = Debug|Win32
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.Build.0 = Debug|Win32
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.ActiveCfg = Debug|x64
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.Build.0 = Debug|x64
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.ActiveCfg = Release|Win32
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.Build.0 = Release|Win32
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.ActiveCfg = Release|x64
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.Build.0 = Release|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.ActiveCfg = Debug|Win32
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.Build.0 = Debug|Win32
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.ActiveCfg = Debug|x64
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.Build.0 = Debug|x64
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.ActiveCfg = Release|Win32
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.Build.0 = Release|Win32
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.ActiveCfg = Release|x64
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.Build.0 = Release|x64
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.ActiveCfg = Debug|Win32
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.Build.0 = Debug|Win32
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.ActiveCfg = Debug|x64
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.Build.0 = Debug|x64
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.ActiveCfg = Release|Win32
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.Build.0 = Release|Win32
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.ActiveCfg = Release|x64
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.Build.0 = Release|x64
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.ActiveCfg = Debug|Win32
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.Build.0 = Debug|Win32
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.ActiveCfg = Debug|x64
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.Build.0 = Debug|x64
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.ActiveCfg = Release|Win32
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.Build.0 = Release|Win32
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.ActiveCfg = Release|x64
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.Build.0 = Release|x64
+		{13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32
+		{13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32
+		{13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64
+		{13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64
+		{13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32
+		{13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32
+		{13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64
+		{13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal
-- 
cgit v0.12


From 6d931b6a93100fe2cb4dafc032ff486c814d1fed Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 12:40:33 -0700
Subject: fixed lz4 compression starting at small address

when using byU32 and byU16 modes
---
 lib/lz4.c      | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---
 tests/fuzzer.c |  2 +-
 2 files changed, 68 insertions(+), 4 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0fdbe5e..bcebc92 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -533,6 +533,20 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_
     LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
 }
 
+/* LZ4_getIndexOnHash() :
+ * Index of match position registered in hash table.
+ * hash position must be calculated by using base+index, or dictBase+index.
+ * Assumption 1 : only valid if tableType == byU32 or byU16.
+ * Assumption 2 : h is presumed valid (within limits of hash table)
+ */
+static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType)
+{
+    LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2);
+    if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-2))); return hashTable[h]; }
+    if (tableType == byU16) { const U16* const hashTable = (const U16*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-1))); return hashTable[h]; }
+    assert(0); return 0;  /* forbidden case */
+}
+
 static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; }
@@ -598,7 +612,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 {
     const BYTE* ip = (const BYTE*) source;
 
-    size_t currentOffset = cctx->currentOffset;
+    size_t const currentOffset = cctx->currentOffset;
     const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
 
@@ -650,7 +664,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         BYTE* token;
 
         /* Find a match */
-        {   const BYTE* forwardIp = ip;
+        if (tableType == byPtr) {
+            const BYTE* forwardIp = ip;
             unsigned step = 1;
             unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
             do {
@@ -687,6 +702,54 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
                 || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
+
+        } else {   /* byU32, byU16 */
+
+            const BYTE* forwardIp = ip;
+            unsigned step = 1;
+            unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
+            do {
+                U32 const h = forwardH;
+                U32 const matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+                ip = forwardIp;
+                forwardIp += step;
+                step = (searchMatchNb++ >> LZ4_skipTrigger);
+
+                if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
+                assert(ip < mflimitPlusOne);
+
+                if (dictDirective == usingDictCtx) {
+                    if (matchIndex < currentOffset) {
+                        /* there was no match, try the dictionary */
+                        match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
+                        refDelta = dictDelta;
+                        lowLimit = dictLowLimit;
+                    } else {
+                        match = base + matchIndex;
+                        refDelta = 0;
+                        lowLimit = (const BYTE*)source;
+                    }
+                } else if (dictDirective==usingExtDict) {
+                    if (matchIndex < currentOffset) {
+                        match = dictBase + matchIndex;
+                        refDelta = dictDelta;
+                        lowLimit = dictLowLimit;
+                    } else {
+                        match = base + matchIndex;
+                        refDelta = 0;
+                        lowLimit = (const BYTE*)source;
+                    }
+                } else {   /* single continuous memory segment */
+                    match = base + matchIndex;
+                    refDelta = 0;
+                    lowLimit = (const BYTE*)source;
+                }
+                forwardH = LZ4_hashPosition(forwardIp, tableType);
+                LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
+
+            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
+                || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
+                || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
         }
 
         /* Catch up */
@@ -718,7 +781,8 @@ _next_match:
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingDictCtx) && lowLimit==dictionary) {
+            if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx)
+              && (lowLimit==dictionary) ) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 2b9b926..0b7d54e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -322,7 +322,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 #   define FUZ_DISPLAYTEST(...) {                 \
                 testNb++;                         \
                 if (g_displayLevel>=4) {          \
-                    printf("\r%4u - %2u ", seed, testNb);  \
+                    printf("\r%4u - %2u ", cycleNb, testNb);  \
                     printf(" " __VA_ARGS__);      \
                     printf("   ");                \
                     fflush(stdout);               \
-- 
cgit v0.12


From 64a3e41acaf9e186937d32c9dd2dc104e5bc4a72 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 16:38:43 -0700
Subject: changed LZ4_compress_generic() logic

to use indexes (U32) instead of Ptr.

byPtr is still present.
---
 lib/lz4.c | 170 ++++++++++++++++++++++++++++++++++++--------------------------
 1 file changed, 100 insertions(+), 70 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index bcebc92..4b219d2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -516,6 +516,18 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab
     return LZ4_hash4(LZ4_read32(p), tableType);
 }
 
+static void LZ4_putIndexOnHash(U32 index, U32 h, void* tableBase, tableType_t const tableType)
+{
+    switch (tableType)
+    {
+    default: /* fallthrough */
+    case clearedTable: /* fallthrough */
+    case byPtr: { /* illegal! */ assert(0); return; }
+    case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = index; return; }
+    case byU16: { U16* hashTable = (U16*) tableBase; assert(index < 65536); hashTable[h] = (U16)index; return; }
+    }
+}
+
 static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
 {
     switch (tableType)
@@ -612,8 +624,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 {
     const BYTE* ip = (const BYTE*) source;
 
-    size_t const currentOffset = cctx->currentOffset;
-    const BYTE* base = (const BYTE*) source - currentOffset;
+    size_t const startIndex = cctx->currentOffset;
+    const BYTE* base = (const BYTE*) source - startIndex;
     const BYTE* lowLimit;
 
     const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
@@ -622,7 +634,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     const U32 dictSize =
         dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
 
-    const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
+    int const maybe_ext_memSegment = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
+    U32 const prefixIdxLimit = startIndex - dictSize;   /* used when dictDirective == dictSmall */
     const BYTE* const dictEnd = dictionary + dictSize;
     const BYTE* anchor = (const BYTE*) source;
     const BYTE* const iend = ip + inputSize;
@@ -633,19 +646,20 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
      * while a dictionary in the current context precedes the currentOffset */
     const BYTE* dictBase = dictDirective == usingDictCtx ?
         (const BYTE*) source - dictCtx->currentOffset :
-        (const BYTE*) source - dictSize - currentOffset;
-    const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
+        (const BYTE*) source - dictSize - startIndex;
     const BYTE* dictLowLimit;
 
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
 
+    U32 offset = 0;
     ptrdiff_t retval = 0;
 
     U32 forwardH;
 
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
+    if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
     dictLowLimit = dictionary ? dictionary : lowLimit;
@@ -659,7 +673,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     /* Main Loop */
     for ( ; ; ) {
-        ptrdiff_t refDelta = 0;
         const BYTE* match;
         BYTE* token;
 
@@ -678,82 +691,65 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 assert(ip < mflimitPlusOne);
 
                 match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
-                if (dictDirective == usingDictCtx) {
-                    if (match < (const BYTE*)source) {
-                        /* there was no match, try the dictionary */
-                        match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
-                        refDelta = dictDelta;
-                        lowLimit = dictLowLimit;
-                    } else {
-                        refDelta = 0;
-                        lowLimit = (const BYTE*)source;
-                    }
-                } else if (dictDirective==usingExtDict) {
-                    if (match < (const BYTE*)source) {
-                        refDelta = dictDelta;
-                        lowLimit = dictLowLimit;
-                    } else {
-                        refDelta = 0;
-                        lowLimit = (const BYTE*)source;
-                }   }
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
 
-            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
-                || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
-                || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
+            } while ( LZ4_read32(match) != LZ4_read32(ip) );
 
         } else {   /* byU32, byU16 */
 
             const BYTE* forwardIp = ip;
-            unsigned step = 1;
             unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
             do {
                 U32 const h = forwardH;
-                U32 const matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+                U32 const current = forwardIp - base;
+                U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+                assert(matchIndex <= current);
                 ip = forwardIp;
-                forwardIp += step;
-                step = (searchMatchNb++ >> LZ4_skipTrigger);
+                assert(searchMatchNb >= (1<<LZ4_skipTrigger));
+                forwardIp += (searchMatchNb++ >> LZ4_skipTrigger);
 
                 if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
                 assert(ip < mflimitPlusOne);
 
                 if (dictDirective == usingDictCtx) {
-                    if (matchIndex < currentOffset) {
+                    if (matchIndex < startIndex) {
                         /* there was no match, try the dictionary */
-                        match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
-                        refDelta = dictDelta;
+                        matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
+                        match = dictBase + matchIndex;
                         lowLimit = dictLowLimit;
                     } else {
                         match = base + matchIndex;
-                        refDelta = 0;
                         lowLimit = (const BYTE*)source;
                     }
                 } else if (dictDirective==usingExtDict) {
-                    if (matchIndex < currentOffset) {
+                    if (matchIndex < startIndex) {
                         match = dictBase + matchIndex;
-                        refDelta = dictDelta;
                         lowLimit = dictLowLimit;
                     } else {
                         match = base + matchIndex;
-                        refDelta = 0;
                         lowLimit = (const BYTE*)source;
                     }
                 } else {   /* single continuous memory segment */
                     match = base + matchIndex;
-                    refDelta = 0;
-                    lowLimit = (const BYTE*)source;
                 }
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
-                LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
+                LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
+
+                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;    /* match outside of valid area */
+                if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue;  /* too far */
+                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);     /* too_far presumed impossible with byU16 */
 
-            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
-                || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
-                || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
+                if (LZ4_read32(match) == LZ4_read32(ip)) {
+                    if (maybe_ext_memSegment) offset = current - matchIndex;
+                    break;   /* match found */
+                }
+
+            } while(1);
         }
 
         /* Catch up */
-        while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; }
+        while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
 
         /* Encode Literals */
         {   unsigned const litLength = (unsigned)(ip - anchor);
@@ -776,7 +772,13 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
 _next_match:
         /* Encode Offset */
-        LZ4_writeLE16(op, (U16)(ip-match)); op+=2;
+        if (maybe_ext_memSegment) {   /* static test */
+            assert(offset <= MAX_DISTANCE && offset > 0);
+            LZ4_writeLE16(op, (U16)offset); op+=2;
+        } else  {
+            assert(ip-match <= MAX_DISTANCE);
+            LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
+        }
 
         /* Encode MatchLength */
         {   unsigned matchCode;
@@ -784,7 +786,6 @@ _next_match:
             if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx)
               && (lowLimit==dictionary) ) {
                 const BYTE* limit;
-                match += refDelta;
                 limit = ip + (dictEnd-match);
                 if (limit > matchlimit) limit = matchlimit;
                 matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit);
@@ -799,6 +800,8 @@ _next_match:
                 ip += MINMATCH + matchCode;
             }
 
+            DEBUGLOG(2,"matchLength:%7u ", matchCode+MINMATCH);
+
             if ( outputLimited &&    /* Check output buffer overflow */
                 (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
                 goto _clean_up;
@@ -825,34 +828,61 @@ _next_match:
         /* Fill table */
         LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);
 
+#if 1
         /* Test next position */
-        match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
-        if (dictDirective == usingDictCtx) {
-            if (match < (const BYTE*)source) {
-                /* there was no match, try the dictionary */
-                match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
-                refDelta = dictDelta;
-                lowLimit = dictLowLimit;
-            } else {
-                refDelta = 0;
-                lowLimit = (const BYTE*)source;
+        if (tableType == byPtr) {
+
+            match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
+            LZ4_putPosition(ip, cctx->hashTable, tableType, base);
+            if ( (match+MAX_DISTANCE >= ip)
+              && (LZ4_read32(match) == LZ4_read32(ip)) )
+            { token=op++; *token=0; goto _next_match; }
+
+        } else {   /* byU32, byU16 */
+
+            U32 const h = LZ4_hashPosition(ip, tableType);
+            U32 const current = (U32)(ip-base);
+            U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+            assert(matchIndex < current);
+            if (dictDirective == usingDictCtx) {
+                if (match < (const BYTE*)source) {
+                    /* there was no match, try the dictionary */
+                    matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
+                    match = dictBase + matchIndex;
+                } else {
+                    match = base + matchIndex;
+                }
+            } else if (dictDirective==usingExtDict) {
+                if (match < (const BYTE*)source) {
+                    match = dictBase + matchIndex;
+                } else {
+                    match = base + matchIndex;
+                }
+            } else {   /* single memory segment */
+                match = base + matchIndex;
             }
-        } else if (dictDirective==usingExtDict) {
-            if (match < (const BYTE*)source) {
-                refDelta = dictDelta;
-                lowLimit = dictLowLimit;
-            } else {
-                refDelta = 0;
-                lowLimit = (const BYTE*)source;
-        }   }
-        LZ4_putPosition(ip, cctx->hashTable, tableType, base);
-        if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
-            && (match+MAX_DISTANCE>=ip)
-            && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
-        { token=op++; *token=0; goto _next_match; }
+            LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
+            if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
+              && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current))
+              && (LZ4_read32(match) == LZ4_read32(ip)) ) {
+                token=op++;
+                *token=0;
+                if (maybe_ext_memSegment)
+                    offset = current - matchIndex;
+                goto _next_match;
+            }
+        }
 
         /* Prepare next loop */
         forwardH = LZ4_hashPosition(++ip, tableType);
+
+#else
+
+        /* Prepare next loop */
+        forwardH = LZ4_hashPosition(ip, tableType);
+
+#endif
+
     }
 
 _last_literals:
-- 
cgit v0.12


From f2a4d6ef37f653c21627274634d171af66126d5e Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 17:16:33 -0700
Subject: fixed immediate match search

---
 lib/lz4.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4b219d2..8d8c1e8 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -624,7 +624,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 {
     const BYTE* ip = (const BYTE*) source;
 
-    size_t const startIndex = cctx->currentOffset;
+    U32 const startIndex = cctx->currentOffset;
     const BYTE* base = (const BYTE*) source - startIndex;
     const BYTE* lowLimit;
 
@@ -645,8 +645,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* the dictCtx currentOffset is indexed on the start of the dictionary,
      * while a dictionary in the current context precedes the currentOffset */
     const BYTE* dictBase = dictDirective == usingDictCtx ?
-        (const BYTE*) source - dictCtx->currentOffset :
-        (const BYTE*) source - dictSize - startIndex;
+        dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */
+        dictionary + dictSize - startIndex;
     const BYTE* dictLowLimit;
 
     BYTE* op = (BYTE*) dest;
@@ -800,8 +800,6 @@ _next_match:
                 ip += MINMATCH + matchCode;
             }
 
-            DEBUGLOG(2,"matchLength:%7u ", matchCode+MINMATCH);
-
             if ( outputLimited &&    /* Check output buffer overflow */
                 (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
                 goto _clean_up;
@@ -845,7 +843,7 @@ _next_match:
             U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
             assert(matchIndex < current);
             if (dictDirective == usingDictCtx) {
-                if (match < (const BYTE*)source) {
+                if (matchIndex < startIndex) {
                     /* there was no match, try the dictionary */
                     matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                     match = dictBase + matchIndex;
@@ -853,7 +851,7 @@ _next_match:
                     match = base + matchIndex;
                 }
             } else if (dictDirective==usingExtDict) {
-                if (match < (const BYTE*)source) {
+                if (matchIndex < startIndex) {
                     match = dictBase + matchIndex;
                 } else {
                     match = base + matchIndex;
-- 
cgit v0.12


From b4be1e0a743f2200eaf1c13d322c925b64b872e2 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 17:52:54 -0700
Subject: fixed byPtr match search

---
 lib/lz4.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 8d8c1e8..5791556 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -694,7 +694,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
 
-            } while ( LZ4_read32(match) != LZ4_read32(ip) );
+            } while ( (match+MAX_DISTANCE < ip)
+                   || (LZ4_read32(match) != LZ4_read32(ip)) );
 
         } else {   /* byU32, byU16 */
 
-- 
cgit v0.12


From f4e06e28e6d285f7f145798d7dfe1cbe71ae1efa Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 18:29:42 -0700
Subject: fixed byPtr mode

switch to byU32 when src address is < 64K

note : byPtr is still useful in 32-bits, as it's about ~10% faster
---
 lib/lz4.c | 47 +++++++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 5791556..4fb54f8 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -930,14 +930,14 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
         if (inputSize < LZ4_64Klimit) {
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {;
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
@@ -951,7 +951,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
  * performance when the context reset would otherwise be a significant part of
  * the cost of the compression, e.g., when the data to be compressed is small.
  */
-int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+int LZ4_compress_fast_extState_noReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration)
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -959,39 +959,39 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
     ctx->dictSize = 0;
     ctx->dictCtx = NULL;
 
-    if (maxOutputSize >= LZ4_compressBound(inputSize)) {
-        if (inputSize < LZ4_64Klimit) {
+    if (dstCapacity >= LZ4_compressBound(srcSize)) {
+        if (srcSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, srcSize, tableType, noDict);
             if (ctx->currentOffset) {
-                return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
-                return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            const tableType_t tableType =  ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
+            LZ4_prepareTable(ctx, srcSize, tableType, noDict);
             if (ctx->currentOffset) {
                 ctx->currentOffset += 64 KB;
             }
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
-        if (inputSize < LZ4_64Klimit) {
+        if (srcSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, srcSize, tableType, noDict);
             if (ctx->currentOffset) {
-                return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
-                return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
+            LZ4_prepareTable(ctx, srcSize, tableType, noDict);
             if (ctx->currentOffset) {
                 ctx->currentOffset += 64 KB;
             }
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
 }
@@ -1098,6 +1098,8 @@ static int LZ4_compress_destSize_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base);
 
+                DEBUGLOG(2, "match:%p , ip:%p", match, ip);
+
             } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match) != LZ4_read32(ip)) );
         }
@@ -1203,11 +1205,12 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
     if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) {  /* compression success is guaranteed */
         return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
     } else {
-        if (*srcSizePtr < LZ4_64Klimit)
+        if (*srcSizePtr < LZ4_64Klimit) {
             return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16);
-        else
-            return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr);
-    }
+        } else {
+            tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
+            return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, tableType);
+    }   }
 }
 
 
-- 
cgit v0.12


From 038a0d95bfe2d3a544aa2f5551998f6fd8bc0722 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 18:39:22 -0700
Subject: added low-memory address test to travis

requires modification linux configuration (sudo)
---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 466d55e..a446420 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,7 +15,7 @@ matrix:
 
     - os: linux
       sudo: false
-      env: Ubu=12.04cont Cmd='make -C tests test-frametest test-fuzzer' COMPILER=cc
+      env: Ubu=12.04cont Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest test-fuzzer' COMPILER=cc
 
     - os: linux
       sudo: false
@@ -59,7 +59,7 @@ matrix:
             - libc6-dev-i386
             - gcc-multilib
 
-    - env: Ubu=14.04 Cmd='make -C tests test-frametest32 test-fuzzer32' COMPILER=cc
+    - env: Ubu=14.04 Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest32 test-fuzzer32' COMPILER=cc
       dist: trusty
       sudo: required
       addons:
-- 
cgit v0.12


From f9992fa37f1b0810c4d0a3e3e6a0eb4880168c57 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 19:05:49 -0700
Subject: noticed a bug when re-using hash table

./fuzzer -vv -s4217 -t7518
---
 lib/lz4.c      |  7 ++++---
 tests/fuzzer.c | 59 +++++++++++++++++++++++++++++-----------------------------
 2 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4fb54f8..54336c7 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -593,6 +593,9 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
             MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
             cctx->currentOffset = 0;
             cctx->tableType = clearedTable;
+        } else {
+            DEBUGLOG(4, "Re-use hash table (no reset)");
+            //if (tableType == byU32) cctx->currentOffset += 64 KB;
         }
     }
     /* If the current offset is zero, we will never look in the external
@@ -1098,8 +1101,6 @@ static int LZ4_compress_destSize_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base);
 
-                DEBUGLOG(2, "match:%p , ip:%p", match, ip);
-
             } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match) != LZ4_read32(ip)) );
         }
@@ -1371,7 +1372,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 LZ4_prepareTable(streamPtr, inputSize, tableType, usingDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
-        } else {
+        } else {  /* no dictCtx */
             LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 0b7d54e..8ffdb22 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -291,7 +291,7 @@ static void FUZ_findDiff(const void* buff1, const void* buff2)
     const BYTE* const b2 = (const BYTE*)buff2;
     size_t u = 0;
     while (b1[u]==b2[u]) u++;
-    DISPLAY("Wrong Byte at position %u \n", (unsigned)u);
+    DISPLAY("\nWrong Byte at position %u \n", (unsigned)u);
 }
 
 
@@ -330,7 +330,6 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
 
     /* init */
-    DISPLAYLEVEL(2, " g_displayLevel = %u \n", g_displayLevel);
     if(!CNBuffer || !compressedBuffer || !decodedBuffer) {
         DISPLAY("Not enough memory to start fuzzer tests");
         goto _output_error;
@@ -360,7 +359,6 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         int compressedSize, HCcompressedSize;
         int blockContinueCompressedSize;
         U32 const crcOrig = XXH32(block, blockSize, 0);
-        U32 crcCheck;
         int ret;
 
         FUZ_displayUpdate(cycleNb);
@@ -467,8 +465,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
         FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
+        }
 
         /* Test decoding with one byte missing => must fail */
         FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
@@ -489,8 +488,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+        }
 
         // Test decoding with more than enough output size => must work
         FUZ_DISPLAYTEST();
@@ -501,8 +501,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
         //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size");   // well, is that an issue ?
         FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+        }
 
         // Test decoding with output size being one byte too short => must fail
         FUZ_DISPLAYTEST();
@@ -605,20 +606,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         memcpy(decodedBuffer, dict, dictSize);
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
-        crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
-        if (crcCheck!=crcOrig) {
-            int i=0;
-            while (block[i]==decodedBuffer[i]) i++;
-            printf("Wrong Byte at position %i/%i\n", i, blockSize);
-
+        {   U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
+            if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
         }
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
 
         FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()");
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
-        crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        }
 
         /* Compress using External dictionary */
         FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary");
@@ -640,22 +638,24 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
 
         /* Decompress with dictionary as external */
-        FUZ_DISPLAYTEST();
+        FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict(): decoding %i bytes, dict(%p) of size %i", blockSize, dict, dictSize);
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+        }
 
         FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        }
 
         FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
@@ -704,10 +704,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        if (crcCheck!=crcOrig)
-            FUZ_findDiff(block, decodedBuffer);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        }
 
         /* Compress HC continue destSize */
         FUZ_DISPLAYTEST();
@@ -729,8 +729,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
             {   U32 const crcSrc = XXH32(block, consumedSize, 0);
                 U32 const crcDst = XXH32(decodedBuffer, consumedSize, 0);
-                if (crcSrc!=crcDst)
-                    FUZ_findDiff(block, decodedBuffer);
+                if (crcSrc!=crcDst) FUZ_findDiff(block, decodedBuffer);
                 FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data");
             }
         }
-- 
cgit v0.12


From 133a50b780be302e726da772592e505d5b143f44 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 6 Apr 2018 14:16:23 -0700
Subject: fixed DISPLAYUPDATE()

wrong comparison, which was always overflowing (hence was always true)
except when it was not (i386, reported by pmc)
in which case it would never show any information.
---
 programs/lz4io.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/programs/lz4io.c b/programs/lz4io.c
index ca13316..6d0d0d0 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -94,9 +94,12 @@
 static int g_displayLevel = 0;   /* 0 : no display  ; 1: errors  ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
 
 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if (((clock_t)(g_time - clock()) > refreshRate) || (g_displayLevel>=4)) \
-            { g_time = clock(); DISPLAY(__VA_ARGS__); \
-            if (g_displayLevel>=4) fflush(stderr); } }
+            if ( ((clock() - g_time) > refreshRate)    \
+              || (g_displayLevel>=4) ) {               \
+                g_time = clock();                      \
+                DISPLAY(__VA_ARGS__);                  \
+                if (g_displayLevel>=4) fflush(stderr); \
+        }   }
 static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
 static clock_t g_time = 0;
 
-- 
cgit v0.12


From 5622c276e114c75f11f6171f9fe2295a8b626e89 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 6 Apr 2018 18:52:55 -0400
Subject: Return to Allowing Early Returns in LZ4_compress_generic()

Or: `goto` Considered Harmful

Or: https://xkcd.com/292/
---
 lib/lz4.c | 38 +++++++++++++++++---------------------
 1 file changed, 17 insertions(+), 21 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0fdbe5e..8743cb9 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -626,8 +626,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
 
-    ptrdiff_t retval = 0;
-
     U32 forwardH;
 
     /* Init conditions */
@@ -637,6 +635,19 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     dictLowLimit = dictionary ? dictionary : lowLimit;
 
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
+
+    /* Update context state */
+    if (dictDirective == usingDictCtx) {
+        /* Subsequent linked blocks can't use the dictionary. */
+        /* Instead, they use the block we just compressed. */
+        cctx->dictCtx = NULL;
+        cctx->dictSize = (U32)inputSize;
+    } else {
+        cctx->dictSize += (U32)inputSize;
+    }
+    cctx->currentOffset += (U32)inputSize;
+    cctx->tableType = tableType;
+
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* First Byte */
@@ -697,7 +708,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             token = op++;
             if ((outputLimited) &&  /* Check output buffer overflow */
                 (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
-                goto _clean_up;
+                return 0;
             if (litLength >= RUN_MASK) {
                 int len = (int)litLength-RUN_MASK;
                 *token = (RUN_MASK<<ML_BITS);
@@ -737,7 +748,7 @@ _next_match:
 
             if ( outputLimited &&    /* Check output buffer overflow */
                 (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
-                goto _clean_up;
+                return 0;
             if (matchCode >= ML_MASK) {
                 *token += ML_MASK;
                 matchCode -= ML_MASK;
@@ -796,7 +807,7 @@ _last_literals:
     {   size_t const lastRun = (size_t)(iend - anchor);
         if ( (outputLimited) &&  /* Check output buffer overflow */
             ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) )
-            goto _clean_up;
+            return 0;
         if (lastRun >= RUN_MASK) {
             size_t accumulator = lastRun - RUN_MASK;
             *op++ = RUN_MASK << ML_BITS;
@@ -809,22 +820,7 @@ _last_literals:
         op += lastRun;
     }
 
-    retval = (((char*)op)-dest);
-
-_clean_up:
-    if (dictDirective == usingDictCtx) {
-        /* Subsequent linked blocks can't use the dictionary. */
-        /* Instead, they use the block we just compressed. */
-        cctx->dictCtx = NULL;
-        cctx->dictSize = (U32)inputSize;
-    } else {
-        cctx->dictSize += (U32)inputSize;
-    }
-    cctx->currentOffset += (U32)inputSize;
-    cctx->tableType = tableType;
-
-    /* End */
-    return (int)retval;
+    return (int)(((char*)op) - dest);
 }
 
 
-- 
cgit v0.12


From f88dc900553b3a029e9cd114800016f9364ffc7b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 6 Apr 2018 16:52:29 -0400
Subject: Avoid Calling LZ4_prepareTable() in LZ4_compress_fast_continue()

---
 lib/lz4.c | 34 ++++++++++++++--------------------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 8743cb9..4f353c5 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -574,11 +574,16 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * indicates a miss. In that case, we need to bump the offset to something
      * non-zero.
      */
-    if (dictDirective == usingDictCtx &&
-        tableType != byPtr &&
-        cctx->currentOffset == 0)
-    {
-        cctx->currentOffset = 1;
+    if (cctx->currentOffset == 0) {
+        if (dictDirective == usingDictCtx) {
+            if (tableType == byU16) {
+                cctx->currentOffset = 1;
+            } else if (tableType == byU32) {
+                cctx->currentOffset = 64 KB;
+            }
+        }
+    } else if (tableType == byU32) {
+        cctx->currentOffset += 64 KB;
     }
 }
 
@@ -874,9 +879,6 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
-            if (ctx->currentOffset) {
-                ctx->currentOffset += 64 KB;
-            }
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
@@ -891,9 +893,6 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
-            if (ctx->currentOffset) {
-                ctx->currentOffset += 64 KB;
-            }
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
@@ -1168,31 +1167,28 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
 int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 {
     LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse;
+    const tableType_t tableType = byU32;
     const BYTE* p = (const BYTE*)dictionary;
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
     DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
 
-    if ((dict->initCheck)
-      || (dict->tableType != byU32 && dict->tableType != clearedTable)
-      || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
-        LZ4_resetStream(LZ4_dict);
+    LZ4_prepareTable(dict, 0, tableType, usingExtDict);
 
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
-    dict->currentOffset += 64 KB;
     base = p - dict->currentOffset;
     dict->dictionary = p;
     dict->dictSize = (U32)(dictEnd - p);
     dict->currentOffset += dict->dictSize;
-    dict->tableType = byU32;
+    dict->tableType = tableType;
 
     if (dictSize < (int)HASH_UNIT) {
         return 0;
     }
 
     while (p <= dictEnd-HASH_UNIT) {
-        LZ4_putPosition(p, dict->hashTable, byU32, base);
+        LZ4_putPosition(p, dict->hashTable, tableType, base);
         p+=3;
     }
 
@@ -1244,7 +1240,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
-        LZ4_prepareTable(streamPtr, inputSize, tableType, withPrefix64k);
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
             return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
@@ -1272,7 +1267,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
         } else {
-            LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             } else {
-- 
cgit v0.12


From cf2f06a6c5046ef7d576bc8ad210c3f2efe1f401 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Mon, 9 Apr 2018 17:08:17 -0700
Subject: fixed minor conversion warning

ptr diff -> U32
---
 lib/lz4.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 3db37b0..a3c2860 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -717,9 +717,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
             do {
                 U32 const h = forwardH;
-                U32 const current = forwardIp - base;
+                U32 const current = (U32)(forwardIp - base);
                 U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
                 assert(matchIndex <= current);
+                assert(forwardIp - base < (ptrdiff_t)(2 GB - 1));
                 ip = forwardIp;
                 assert(searchMatchNb >= (1<<LZ4_skipTrigger));
                 forwardIp += (searchMatchNb++ >> LZ4_skipTrigger);
-- 
cgit v0.12


From ad7e040384d1835c097b0f727aeaa3d598a401b6 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Mon, 9 Apr 2018 20:38:00 -0700
Subject: fix minor conversion warning

cast from void not implicit for C++
---
 tests/fuzzer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 8ffdb22..2e3ee92 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -367,7 +367,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         if ( ((FUZ_rand(&randState) & 63) == 2)
           && ((size_t)blockSize < labSize) ) {
             memcpy(lowAddrBuffer, block, blockSize);
-            block = lowAddrBuffer;
+            block = (const char*)lowAddrBuffer;
         }
 
         /* Test compression destSize */
-- 
cgit v0.12


From 59c7d951213bf8593438ae811549655d5a78d2fc Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 10 Apr 2018 13:12:30 -0400
Subject: Expose a Faster Stream Reset Function

---
 lib/lz4.c      | 53 ++++++++++++++++++++++++++++-------------------------
 lib/lz4.h      |  8 ++++++++
 lib/lz4frame.c |  4 +---
 3 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4f353c5..30146bb 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -546,12 +546,10 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
 }
 
-
 LZ4_FORCE_INLINE void LZ4_prepareTable(
         LZ4_stream_t_internal* const cctx,
         const int inputSize,
-        const tableType_t tableType,
-        const dict_directive dictDirective) {
+        const tableType_t tableType) {
     /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
      * therefore safe to use no matter what mode we're in. Otherwise, we figure
      * out if it's safe to leave as is or whether it needs to be reset.
@@ -569,22 +567,23 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
             cctx->tableType = clearedTable;
         }
     }
-    /* If the current offset is zero, we will never look in the external
-     * dictionary context, since there is no value a table entry can take that
-     * indicates a miss. In that case, we need to bump the offset to something
-     * non-zero.
+
+    /* Adding a gap, so all previous entries are > MAX_DISTANCE back, is faster
+     * than compressing without a gap. However, compressing with
+     * currentOffset == 0 is faster still, so we preserve that case.
      */
-    if (cctx->currentOffset == 0) {
-        if (dictDirective == usingDictCtx) {
-            if (tableType == byU16) {
-                cctx->currentOffset = 1;
-            } else if (tableType == byU32) {
-                cctx->currentOffset = 64 KB;
-            }
-        }
-    } else if (tableType == byU32) {
+    if (cctx->currentOffset != 0 && tableType == byU32) {
         cctx->currentOffset += 64 KB;
     }
+
+    /* Finally, clear history */
+    cctx->dictCtx = NULL;
+    cctx->dictionary = NULL;
+    cctx->dictSize = 0;
+}
+
+void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
+    LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
 }
 
 /** LZ4_compress_generic() :
@@ -863,14 +862,11 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
-    ctx->dictionary = NULL;
-    ctx->dictSize = 0;
-    ctx->dictCtx = NULL;
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType);
             if (ctx->currentOffset) {
                 return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -878,13 +874,13 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType);
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType);
             if (ctx->currentOffset) {
                 return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -892,7 +888,7 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType);
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
@@ -1174,7 +1170,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 
     DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
 
-    LZ4_prepareTable(dict, 0, tableType, usingExtDict);
+    LZ4_prepareTable(dict, 0, tableType);
 
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
     base = p - dict->currentOffset;
@@ -1263,7 +1259,14 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                LZ4_prepareTable(streamPtr, inputSize, tableType, usingDictCtx);
+                /* If the current offset is zero, we will never look in the
+                 * external dictionary context, since there is no value a table
+                 * entry can take that indicate a miss. In that case, we need
+                 * to bump the offset to something non-zero.
+                 */
+                if (streamPtr->currentOffset == 0) {
+                    streamPtr->currentOffset = 64 KB;
+                }
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
         } else {
diff --git a/lib/lz4.h b/lib/lz4.h
index 80f040f..9c1b694 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -366,6 +366,14 @@ LZ4_compress_fast_extState_noReset() :
 LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
+
+/*! LZ4_resetStream_fast() :
+ *  An LZ4_stream_t structure can be allocated once and re-used multiple times.
+ *  Use this function to start compressing a new stream.
+ */
+LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
+
+
 /*-************************************
  *  Private definitions
  **************************************
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a080fa6..afa5af3 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -532,9 +532,7 @@ static void LZ4F_applyCDict(void* ctx,
     if (level < LZ4HC_CLEVEL_MIN) {
         LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
         assert(!internal_ctx->initCheck);
-        /* Clear any local dictionary */
-        internal_ctx->dictionary = NULL;
-        internal_ctx->dictSize = 0;
+        LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         /* Point to the dictionary context */
         internal_ctx->dictCtx = cdict ? &(cdict->fastCtx->internal_donotuse) : NULL;
     } else {
-- 
cgit v0.12


From c18bff933b18106ea000868ce13f37f0589cb58e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 15:12:34 -0400
Subject: Remove Extraneous Assignment (clearedTable == 0)

---
 lib/lz4.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 30146bb..f79ee2c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1147,7 +1147,6 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-    LZ4_stream->internal_donotuse.tableType = clearedTable;
 }
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
-- 
cgit v0.12


From 21f0c9700b28301bdc8b49fcbe4ccdddf4cf5dea Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 15:13:01 -0400
Subject: Rename _extState_noReset -> _extState_fastReset and Edit Comments

---
 lib/lz4.c      | 23 ++++++++++++-----------
 lib/lz4.h      | 43 ++++++++++++++++++++++++++++---------------
 lib/lz4frame.c |  2 +-
 3 files changed, 41 insertions(+), 27 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index f79ee2c..c2d1c32 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -582,10 +582,6 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
     cctx->dictSize = 0;
 }
 
-void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
-    LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
-}
-
 /** LZ4_compress_generic() :
     inlined, to ensure branches are decided at compilation time */
 LZ4_FORCE_INLINE int LZ4_compress_generic(
@@ -851,14 +847,15 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
 }
 
 /**
- * LZ4_compress_fast_extState_noReset is a variant of LZ4_compress_fast_extState
- * that can be used when the state is known to have already been initialized
- * (via LZ4_resetStream or an earlier call to LZ4_compress_fast_extState /
- * LZ4_compress_fast_extState_noReset). This can provide significantly better
- * performance when the context reset would otherwise be a significant part of
- * the cost of the compression, e.g., when the data to be compressed is small.
+ * LZ4_compress_fast_extState_fastReset() :
+ * A variant of LZ4_compress_fast_extState().
+ *
+ * Using this variant avoids an expensive initialization step. It is only safe
+ * to call if the state buffer is known to be correctly initialized already
+ * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of
+ * "correctly initialized").
  */
-int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -1149,6 +1146,10 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
 }
 
+void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
+    LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
+}
+
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
 {
     if (!LZ4_stream) return 0;   /* support free on NULL */
diff --git a/lib/lz4.h b/lib/lz4.h
index 9c1b694..427beaf 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -353,26 +353,39 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
  * Their signatures may change.
  **************************************/
 
-/*!
-LZ4_compress_fast_extState_noReset() :
-    A variant of LZ4_compress_fast_extState().
-
-    Use the _noReset variant if LZ4_resetStream() was called on the state
-    buffer before being used for the first time (calls to both extState
-    functions leave the state in a safe state, so zeroing is not required
-    between calls). Otherwise, using the plain _extState requires LZ4 to
-    reinitialize the state internally for every call.
-*/
-LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
-
-
 
 /*! LZ4_resetStream_fast() :
- *  An LZ4_stream_t structure can be allocated once and re-used multiple times.
- *  Use this function to start compressing a new stream.
+ *  When an LZ4_stream_t is known to be in a internally coherent state,
+ *  it can often be prepared for a new compression with almost no work, only
+ *  sometimes falling back to the full, expensive reset that is always required
+ *  when the stream is in an indeterminate state (i.e., the reset performed by
+ *  LZ4_resetStream()).
+ *
+ *  LZ4_streams are guaranteed to be in a valid state when:
+ *  - returned from LZ4_createStream()
+ *  - reset by LZ4_resetStream()
+ *  - memset(stream, 0, sizeof(LZ4_stream_t))
+ *  - the stream was in a valid state and was reset by LZ4_resetStream_fast()
+ *  - the stream was in a valid state and was then used in any compression call
+ *    that returned success
+ *  - the stream was in an indeterminate state and was used in a compression
+ *    call that fully reset the state (LZ4_compress_fast_extState()) and that
+ *    returned success
  */
 LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
 
+/*! LZ4_compress_fast_extState_fastReset() :
+ *  A variant of LZ4_compress_fast_extState().
+ *
+ *  Using this variant avoids an expensive initialization step. It is only safe
+ *  to call if the state buffer is known to be correctly initialized already
+ *  (see above comment on LZ4_resetStream_fast() for a definition of "correctly
+ *  initialized"). From a high level, the difference is that this function
+ *  initializes the provided state with a call to LZ4_resetStream_fast() while
+ *  LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
+ */
+LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+
 
 /*-************************************
  *  Private definitions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index afa5af3..5e11ba2 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -714,7 +714,7 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize
     if (cdict) {
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
-        return LZ4_compress_fast_extState_noReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
+        return LZ4_compress_fast_extState_fastReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
 }
 
-- 
cgit v0.12


From afa52c9b95dd1f83d80cd9a783a408822700e2d3 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 16:04:24 -0400
Subject: Expose dictCtx Functionality in LZ4

---
 lib/lz4.c      |  4 ++++
 lib/lz4.h      | 28 ++++++++++++++++++++++++++++
 lib/lz4frame.c |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c2d1c32..cee85ec 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1191,6 +1191,10 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     return dict->dictSize;
 }
 
+void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) {
+    working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL;
+}
+
 
 static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
 {
diff --git a/lib/lz4.h b/lib/lz4.h
index 427beaf..f90120e 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -386,6 +386,34 @@ LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
  */
 LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
+/*! LZ4_attach_dictionary() :
+ *  This is an experimental API that allows for the efficient use of a
+ *  static dictionary many times.
+ *
+ *  Rather than re-loading the dictionary buffer into a working context before
+ *  each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
+ *  working LZ4_stream_t, this function introduces a no-copy setup mechanism,
+ *  in which the working stream references the dictionary stream in-place.
+ *
+ *  Several assumptions are made about the state of the dictionary stream.
+ *  Currently, only streams which have been prepared by LZ4_loadDict() should
+ *  be expected to work.
+ *
+ *  Alternatively, the provided dictionary stream pointer may be NULL, in which
+ *  case any existing dictionary stream is unset.
+ *
+ *  If a dictionary is provided, it replaces any pre-existing stream history.
+ *  The dictionary contents are the only history that can be referenced and
+ *  logically immediately precede the data compressed in the first subsequent
+ *  compression call.
+ *
+ *  The dictionary will only remain attached to the working stream through the
+ *  first compression call, at the end of which it is cleared. The dictionary
+ *  stream (and source buffer) must remain in-place / accessible / unchanged
+ *  through the completion of the first compression call on the stream.
+ */
+LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream);
+
 
 /*-************************************
  *  Private definitions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 5e11ba2..d973b5f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -533,8 +533,7 @@ static void LZ4F_applyCDict(void* ctx,
         LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
         assert(!internal_ctx->initCheck);
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
-        /* Point to the dictionary context */
-        internal_ctx->dictCtx = cdict ? &(cdict->fastCtx->internal_donotuse) : NULL;
+        LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
         if (cdict) {
             memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
-- 
cgit v0.12


From 3a0c571272731166774405e1f96b3161827ca09f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 16:31:52 -0400
Subject: Add a LZ4_STATIC_LINKING_ONLY Macro to Guard Experimental APIs

---
 lib/lz4.c      | 1 +
 lib/lz4.h      | 2 ++
 lib/lz4frame.c | 1 +
 3 files changed, 4 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index cee85ec..9e6b1e2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -89,6 +89,7 @@
 /*-************************************
 *  Dependency
 **************************************/
+#define LZ4_STATIC_LINKING_ONLY
 #include "lz4.h"
 /* see also "memory routines" below */
 
diff --git a/lib/lz4.h b/lib/lz4.h
index f90120e..0dfa19e 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -353,6 +353,7 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
  * Their signatures may change.
  **************************************/
 
+#ifdef LZ4_STATIC_LINKING_ONLY
 
 /*! LZ4_resetStream_fast() :
  *  When an LZ4_stream_t is known to be in a internally coherent state,
@@ -414,6 +415,7 @@ LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* sr
  */
 LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream);
 
+#endif
 
 /*-************************************
  *  Private definitions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d973b5f..7608d90 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -74,6 +74,7 @@ You can contact the author at :
 *  Includes
 **************************************/
 #include "lz4frame_static.h"
+#define LZ4_STATIC_LINKING_ONLY
 #include "lz4.h"
 #define LZ4_HC_STATIC_LINKING_ONLY
 #include "lz4hc.h"
-- 
cgit v0.12


From 51a56c47c0d34b6255bdd8e7b33f0ccb44b80351 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 16:55:12 -0400
Subject: Minor Fixes

---
 lib/lz4.c      | 22 +++++++++++++---------
 lib/lz4frame.c |  2 --
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 9e6b1e2..111085a 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1193,7 +1193,19 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 }
 
 void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) {
-    working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL;
+    if (dictionary_stream != NULL) {
+        /* If the current offset is zero, we will never look in the
+         * external dictionary context, since there is no value a table
+         * entry can take that indicate a miss. In that case, we need
+         * to bump the offset to something non-zero.
+         */
+        if (working_stream->internal_donotuse.currentOffset == 0) {
+            working_stream->internal_donotuse.currentOffset = 64 KB;
+        }
+        working_stream->internal_donotuse.dictCtx = &(dictionary_stream->internal_donotuse);
+    } else {
+        working_stream->internal_donotuse.dictCtx = NULL;
+    }
 }
 
 
@@ -1264,14 +1276,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                /* If the current offset is zero, we will never look in the
-                 * external dictionary context, since there is no value a table
-                 * entry can take that indicate a miss. In that case, we need
-                 * to bump the offset to something non-zero.
-                 */
-                if (streamPtr->currentOffset == 0) {
-                    streamPtr->currentOffset = 64 KB;
-                }
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
         } else {
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 7608d90..507e4fe 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -531,8 +531,6 @@ static void LZ4F_applyCDict(void* ctx,
                             const LZ4F_CDict* cdict,
                             int level) {
     if (level < LZ4HC_CLEVEL_MIN) {
-        LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
-        assert(!internal_ctx->initCheck);
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
-- 
cgit v0.12


From 056ea63215800c9c16e81db92eacc9a9c87548ae Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 18:42:09 -0400
Subject: Fix Silly Warning (const-ness in declaration has no effect on value
 types!)

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 111085a..4b0efb1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1147,7 +1147,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
 }
 
-void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
+void LZ4_resetStream_fast(LZ4_stream_t* ctx) {
     LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
 }
 
-- 
cgit v0.12


From 7b3cd10579fdc3d62bbf5e00f07ca19068a4403b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 11 Apr 2018 16:31:43 -0700
Subject: reduced test time on circle-ci

---
 Makefile       |  4 ++--
 circle.yml     | 14 +++++++-------
 tests/Makefile | 13 ++++++-------
 3 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/Makefile b/Makefile
index 3a95808..86613fd 100644
--- a/Makefile
+++ b/Makefile
@@ -143,10 +143,10 @@ clangtest-native: clean
 	@CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) native CC=clang
 
 usan: clean
-	CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1
+	CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
 
 usan32: clean
-	CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1
+	CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
 
 staticAnalyze: clean
 	CFLAGS=-g scan-build --status-bugs -v $(MAKE) all
diff --git a/circle.yml b/circle.yml
index 9a0d1ec..aeb52e9 100644
--- a/circle.yml
+++ b/circle.yml
@@ -11,10 +11,10 @@ test:
     - clang -v; make clangtest && make clean
     - g++ -v; make gpptest     && make clean
     - gcc -v; make c_standards && make clean
-    - gcc-5 -v; make -C tests test-lz4 CC=gcc-5 MOREFLAGS=-Werror && make clean
-    - gcc-5 -v; make -C tests test-lz4c32 CC=gcc-5 MOREFLAGS="-I/usr/include/x86_64-linux-gnu -Werror" && make clean
-    - gcc-6 -v; make c_standards CC=gcc-6 && make clean
-    - gcc-6 -v; make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean
+    - gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean
+    - gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
+    - gcc-6 -v; CC=gcc-6 make c_standards && make clean
+    - gcc-6 -v; CC=gcc-6 MOREFLAGS="-O2 -Werror" make check  && make clean
 # Shorter tests
     - make cmake               && make clean
     - make -C tests test-lz4
@@ -22,11 +22,11 @@ test:
     - make -C tests test-frametest
     - make -C tests test-fullbench
     - make -C tests test-fuzzer && make clean
-    - make -C lib all && make clean
-    - pyenv global 3.4.4; CFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
+    - make -C lib all          && make clean
+    - pyenv global 3.4.4; CPPFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
     - make travis-install      && make clean
   # Longer tests
-    - gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
+    - gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
     - make usan                && make clean
     - clang -v; make staticAnalyze && make clean
   # Valgrind tests
diff --git a/tests/Makefile b/tests/Makefile
index 77f5d02..d11b7fe 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -56,7 +56,7 @@ LZ4 := $(PRGDIR)/lz4$(EXT)
 
 # Default test parameters
 TEST_FILES   := COPYING
-FUZZER_TIME  := -T3mn
+FUZZER_TIME  := -T90s
 NB_LOOPS     ?= -i1
 
 
@@ -393,13 +393,12 @@ test-mem: lz4 datagen fuzzer frametest fullbench
 	./datagen -g16KB -s2 > ftmdg16K2
 	./datagen -g16KB -s3 > ftmdg16K3
 	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3
-	./datagen -g16MB > ftmdg16M
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg16M ftmdg16K2
+	./datagen -g7MB > ftmdg7M
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2
 	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg16M
-	valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg16M ftmdg16K2
-	./datagen -g256MB > ftmdg256M
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg256M $(VOID)
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M
+	valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg7M ftmdg16K2
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg7M $(VOID)
 	$(RM) ftm*
 	valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
 	valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256
-- 
cgit v0.12


From ca388790e689ddb09e5e5f7f86a5bdb17dee67d7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 11 Apr 2018 16:41:25 -0700
Subject: allow system-defined CPPFLAGS in /tests

---
 tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index d11b7fe..25f00b8 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -39,7 +39,7 @@ CFLAGS  += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
            -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
            -Wpointer-arith -Wstrict-aliasing=1
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
-CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
+CPPFLAGS+= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
 FLAGS    = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 
 
-- 
cgit v0.12


From 1838803948ceaf211b9e79fb405ef9b7340762ce Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Wed, 11 Apr 2018 16:49:40 -0700
Subject: fixed LZ4_compress_fast_extState_fastReset()

---
 lib/lz4.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d79b2cc..d564ddc 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -953,7 +953,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
  * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of
  * "correctly initialized").
  */
-int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration)
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -961,8 +961,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char*
     if (dstCapacity >= LZ4_compressBound(srcSize)) {
         if (srcSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType);
-LZ4_prepareTable>>> dev
+            LZ4_prepareTable(ctx, srcSize, tableType);
             if (ctx->currentOffset) {
                 return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -970,13 +969,13 @@ LZ4_prepareTable>>> dev
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+            LZ4_prepareTable(ctx, srcSize, tableType);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (srcSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType);
+            LZ4_prepareTable(ctx, srcSize, tableType);
             if (ctx->currentOffset) {
                 return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -984,8 +983,8 @@ LZ4_prepareTable>>> dev
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            LZ4_prepareTable(ctx, srcSize, tableType);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
 }
-- 
cgit v0.12


From d6d9bccd537e307cc0bf3f8475801dea79e97367 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 12 Apr 2018 06:47:27 -0700
Subject: modified versionsTest

to use MOREFLAGS rather CPPFLAGS
as some older versions of LZ4 overwrite CPPFLAGS environment variable.
---
 circle.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/circle.yml b/circle.yml
index aeb52e9..fa37590 100644
--- a/circle.yml
+++ b/circle.yml
@@ -23,7 +23,7 @@ test:
     - make -C tests test-fullbench
     - make -C tests test-fuzzer && make clean
     - make -C lib all          && make clean
-    - pyenv global 3.4.4; CPPFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
+    - pyenv global 3.4.4; make versionsTest MOREFLAGS=-I/usr/include/x86_64-linux-gnu && make clean
     - make travis-install      && make clean
   # Longer tests
     - gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
-- 
cgit v0.12


From 8af32ce6f7109ecf5cd7d73527e0aba3a63b55e5 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 12 Apr 2018 07:25:40 -0700
Subject: modified a few traces for debug

---
 lib/lz4.c      | 5 +++--
 lib/lz4hc.c    | 7 +++----
 tests/fuzzer.c | 4 +++-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d564ddc..e4a68cd 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -594,7 +594,6 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
             cctx->tableType = clearedTable;
         } else {
             DEBUGLOG(4, "Re-use hash table (no reset)");
-            //if (tableType == byU32) cctx->currentOffset += 64 KB;
         }
     }
 
@@ -602,9 +601,11 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * than compressing without a gap. However, compressing with
      * currentOffset == 0 is faster still, so we preserve that case.
      */
+     DEBUGLOG(2, "tableType=%u, currentOffset=%u", cctx->tableType, cctx->currentOffset);
     if (cctx->currentOffset != 0 && tableType == byU32) {
         cctx->currentOffset += 64 KB;
     }
+    DEBUGLOG(2, "currentOffset: %u", cctx->currentOffset);
 
     /* Finally, clear history */
     cctx->dictCtx = NULL;
@@ -1267,7 +1268,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
-    DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
+    DEBUGLOG(4, "LZ4_loadDict (%p into %p)", dictionary, LZ4_dict);
 
     LZ4_prepareTable(dict, 0, tableType);
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d05760b..111a09b 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -336,7 +336,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;
     U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
     if (start==NULL) start = *anchor;  /* only works for single segment */
-    //g_debuglog_enable = (pos >= 2228) & (pos <= 2262);
+    /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */
     DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",
                 pos,
                 (U32)(*ip - *anchor), matchLength, (U32)(*ip-match),
@@ -1137,15 +1137,14 @@ static int LZ4HC_compress_optimal (
  encode: /* cur, last_match_pos, best_mlen, best_off must be set */
          assert(cur < LZ4_OPT_NUM);
          assert(last_match_pos >= 1);  /* == 1 when only one candidate */
-         DEBUGLOG(6, "reverse traversal, looking for shortest path")
-         DEBUGLOG(6, "last_match_pos = %i", last_match_pos);
+         DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos);
          {   int candidate_pos = cur;
              int selected_matchLength = best_mlen;
              int selected_offset = best_off;
              while (1) {  /* from end to beginning */
                  int const next_matchLength = opt[candidate_pos].mlen;  /* can be 1, means literal */
                  int const next_offset = opt[candidate_pos].off;
-                 DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
+                 DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
                  opt[candidate_pos].mlen = selected_matchLength;
                  opt[candidate_pos].off = selected_offset;
                  selected_matchLength = next_matchLength;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 2e3ee92..a650277 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -632,13 +632,15 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
 
         FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()");
+        DISPLAYLEVEL(5, " compress %i bytes from buffer(%p) into dst(%p) using dict(%p) of size %i \n", blockSize, block, decodedBuffer, dict, dictSize);
         LZ4_loadDict(&LZ4dict, dict, dictSize);
         ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
         FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
 
         /* Decompress with dictionary as external */
-        FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict(): decoding %i bytes, dict(%p) of size %i", blockSize, dict, dictSize);
+        FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as extDict");
+        DISPLAYLEVEL(5, " decoding %i bytes from buffer(%p) using dict(%p) of size %i \n", blockSize, decodedBuffer, dict, dictSize);
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
-- 
cgit v0.12


From db9aa785c51bbcae50c777e89fb537393bfca856 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 12 Apr 2018 16:12:21 -0700
Subject: fixed : counting matches which overlap extDict and prefix

---
 lib/lz4.c | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index e4a68cd..21892d3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -601,11 +601,9 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * than compressing without a gap. However, compressing with
      * currentOffset == 0 is faster still, so we preserve that case.
      */
-     DEBUGLOG(2, "tableType=%u, currentOffset=%u", cctx->tableType, cctx->currentOffset);
     if (cctx->currentOffset != 0 && tableType == byU32) {
         cctx->currentOffset += 64 KB;
     }
-    DEBUGLOG(2, "currentOffset: %u", cctx->currentOffset);
 
     /* Finally, clear history */
     cctx->dictCtx = NULL;
@@ -652,7 +650,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     const BYTE* dictBase = dictDirective == usingDictCtx ?
         dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */
         dictionary + dictSize - startIndex;
-    const BYTE* dictLowLimit;
 
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
@@ -665,7 +662,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
-    dictLowLimit = dictionary ? dictionary : lowLimit;
 
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
 
@@ -735,15 +731,16 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                         /* there was no match, try the dictionary */
                         matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                         match = dictBase + matchIndex;
-                        lowLimit = dictLowLimit;
+                        lowLimit = dictionary;
                     } else {
                         match = base + matchIndex;
                         lowLimit = (const BYTE*)source;
                     }
                 } else if (dictDirective==usingExtDict) {
                     if (matchIndex < startIndex) {
+                        DEBUGLOG(7, "extDict candidate: matchIndex=%5u  <  startIndex=%5u", matchIndex, startIndex);
                         match = dictBase + matchIndex;
-                        lowLimit = dictLowLimit;
+                        lowLimit = dictionary;
                     } else {
                         match = base + matchIndex;
                         lowLimit = (const BYTE*)source;
@@ -786,6 +783,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             /* Copy Literals */
             LZ4_wildCopy(op, anchor, op+litLength);
             op+=litLength;
+            DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, litLength, ip-(const BYTE*)source);
         }
 
 _next_match:
@@ -793,29 +791,33 @@ _next_match:
         if (maybe_ext_memSegment) {   /* static test */
             assert(offset <= MAX_DISTANCE && offset > 0);
             LZ4_writeLE16(op, (U16)offset); op+=2;
+            DEBUGLOG(6, "                with offset=%u  (ext if > %zi)", offset, ip - (const BYTE*)source);
         } else  {
             assert(ip-match <= MAX_DISTANCE);
             LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
+            DEBUGLOG(6, "                with offset=%u  (same segment)", (U32)(ip - match));
         }
 
         /* Encode MatchLength */
         {   unsigned matchCode;
 
             if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx)
-              && (lowLimit==dictionary) ) {
-                const BYTE* limit;
-                limit = ip + (dictEnd-match);
+              && (lowLimit==dictionary) /* match within extDict */ ) {
+                const BYTE* limit = ip + (dictEnd-match);
+                assert(dictEnd > match);
                 if (limit > matchlimit) limit = matchlimit;
                 matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit);
                 ip += MINMATCH + matchCode;
                 if (ip==limit) {
-                    unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit);
+                    unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit);
                     matchCode += more;
                     ip += more;
                 }
+                DEBUGLOG(6, "                with matchLength=%u starting in extDict", matchCode+MINMATCH);
             } else {
                 matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
                 ip += MINMATCH + matchCode;
+                DEBUGLOG(6, "                with matchLength=%u", matchCode+MINMATCH);
             }
 
             if ( outputLimited &&    /* Check output buffer overflow */
@@ -865,14 +867,18 @@ _next_match:
                     /* there was no match, try the dictionary */
                     matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                     match = dictBase + matchIndex;
+                    lowLimit = dictionary;   /* required for match length counter */
                 } else {
                     match = base + matchIndex;
+                    lowLimit = (const BYTE*)source;  /* required for match length counter */
                 }
             } else if (dictDirective==usingExtDict) {
                 if (matchIndex < startIndex) {
                     match = dictBase + matchIndex;
+                    lowLimit = dictionary;   /* required for match length counter */
                 } else {
                     match = base + matchIndex;
+                    lowLimit = (const BYTE*)source;   /* required for match length counter */
                 }
             } else {   /* single memory segment */
                 match = base + matchIndex;
@@ -885,6 +891,7 @@ _next_match:
                 *token=0;
                 if (maybe_ext_memSegment)
                     offset = current - matchIndex;
+                DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, 0, ip-(const BYTE*)source);
                 goto _next_match;
             }
         }
-- 
cgit v0.12


From 98811d606808da9f59affd990670ba34e5a9bee2 Mon Sep 17 00:00:00 2001
From: Cyan4973 <Kdo4973@hotmail.com>
Date: Fri, 13 Apr 2018 00:59:27 -0700
Subject: added sudo rights for low-mem-address tests

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index a446420..0a876f9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,7 +14,7 @@ matrix:
       env: Ubu=12.04cont Cmd='make -C tests test-lz4 test-lz4c test-fullbench' COMPILER=cc
 
     - os: linux
-      sudo: false
+      sudo: required
       env: Ubu=12.04cont Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest test-fuzzer' COMPILER=cc
 
     - os: linux
-- 
cgit v0.12


From 57afa36795f478d0f9b069ad19b578761e3fb16a Mon Sep 17 00:00:00 2001
From: Cyan4973 <Kdo4973@hotmail.com>
Date: Fri, 13 Apr 2018 01:01:54 -0700
Subject: compatibility with gcc-4.4 string.h version

Someone found it would be a great idea to define there a global variable under the very generic name "index".
Cause problem with shadow warnings, so no variable can be named "index" now ...

Also : automatically update API manual
---
 doc/lz4_manual.html | 94 ++++++++++++++++++++++++++++++++++++++++-------------
 lib/lz4.c           |  6 ++--
 2 files changed, 75 insertions(+), 25 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index d14467f..f8639fe 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -15,8 +15,9 @@
 <li><a href="#Chapter5">Advanced Functions</a></li>
 <li><a href="#Chapter6">Streaming Compression Functions</a></li>
 <li><a href="#Chapter7">Streaming Decompression Functions</a></li>
-<li><a href="#Chapter8">Private definitions</a></li>
-<li><a href="#Chapter9">Obsolete Functions</a></li>
+<li><a href="#Chapter8">Unstable declarations</a></li>
+<li><a href="#Chapter9">Private definitions</a></li>
+<li><a href="#Chapter10">Obsolete Functions</a></li>
 </ol>
 <hr>
 <a name="Chapter1"></a><h2>Introduction</h2><pre>
@@ -245,22 +246,80 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
  
 </p></pre><BR>
 
-<a name="Chapter8"></a><h2>Private definitions</h2><pre>
+<a name="Chapter8"></a><h2>Unstable declarations</h2><pre>
+ Declarations in this section should be considered unstable.
+ Use at your own peril, etc., etc.
+ They may be removed in the future.
+ Their signatures may change.
+<BR></pre>
+
+<pre><b>void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
+</b><p>  When an LZ4_stream_t is known to be in a internally coherent state,
+  it can often be prepared for a new compression with almost no work, only
+  sometimes falling back to the full, expensive reset that is always required
+  when the stream is in an indeterminate state (i.e., the reset performed by
+  LZ4_resetStream()).
+
+  LZ4_streams are guaranteed to be in a valid state when:
+  - returned from LZ4_createStream()
+  - reset by LZ4_resetStream()
+  - memset(stream, 0, sizeof(LZ4_stream_t))
+  - the stream was in a valid state and was reset by LZ4_resetStream_fast()
+  - the stream was in a valid state and was then used in any compression call
+    that returned success
+  - the stream was in an indeterminate state and was used in a compression
+    call that fully reset the state (LZ4_compress_fast_extState()) and that
+    returned success
+ 
+</p></pre><BR>
+
+<pre><b>int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+</b><p>  A variant of LZ4_compress_fast_extState().
+
+  Using this variant avoids an expensive initialization step. It is only safe
+  to call if the state buffer is known to be correctly initialized already
+  (see above comment on LZ4_resetStream_fast() for a definition of "correctly
+  initialized"). From a high level, the difference is that this function
+  initializes the provided state with a call to LZ4_resetStream_fast() while
+  LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
+ 
+</p></pre><BR>
+
+<pre><b>void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream);
+</b><p>  This is an experimental API that allows for the efficient use of a
+  static dictionary many times.
+
+  Rather than re-loading the dictionary buffer into a working context before
+  each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
+  working LZ4_stream_t, this function introduces a no-copy setup mechanism,
+  in which the working stream references the dictionary stream in-place.
+
+  Several assumptions are made about the state of the dictionary stream.
+  Currently, only streams which have been prepared by LZ4_loadDict() should
+  be expected to work.
+
+  Alternatively, the provided dictionary stream pointer may be NULL, in which
+  case any existing dictionary stream is unset.
+
+  If a dictionary is provided, it replaces any pre-existing stream history.
+  The dictionary contents are the only history that can be referenced and
+  logically immediately precede the data compressed in the first subsequent
+  compression call.
+
+  The dictionary will only remain attached to the working stream through the
+  first compression call, at the end of which it is cleared. The dictionary
+  stream (and source buffer) must remain in-place / accessible / unchanged
+  through the completion of the first compression call on the stream.
+ 
+</p></pre><BR>
+
+<a name="Chapter9"></a><h2>Private definitions</h2><pre>
  Do not use these definitions.
  They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
  Using these definitions will expose code to API and/or ABI break in future versions of the library.
 <BR></pre>
 
 <pre><b>typedef struct {
-    uint32_t hashTable[LZ4_HASH_SIZE_U32];
-    uint32_t currentOffset;
-    uint32_t initCheck;
-    const uint8_t* dictionary;
-    uint8_t* bufferStart;   </b>/* obsolete, used for slideInputBuffer */<b>
-    uint32_t dictSize;
-} LZ4_stream_t_internal;
-</b></pre><BR>
-<pre><b>typedef struct {
     const uint8_t* externalDict;
     size_t extDictSize;
     const uint8_t* prefixEnd;
@@ -268,15 +327,6 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
 } LZ4_streamDecode_t_internal;
 </b></pre><BR>
 <pre><b>typedef struct {
-    unsigned int hashTable[LZ4_HASH_SIZE_U32];
-    unsigned int currentOffset;
-    unsigned int initCheck;
-    const unsigned char* dictionary;
-    unsigned char* bufferStart;   </b>/* obsolete, used for slideInputBuffer */<b>
-    unsigned int dictSize;
-} LZ4_stream_t_internal;
-</b></pre><BR>
-<pre><b>typedef struct {
     const unsigned char* externalDict;
     size_t extDictSize;
     const unsigned char* prefixEnd;
@@ -311,7 +361,7 @@ union LZ4_streamDecode_u {
  
 </p></pre><BR>
 
-<a name="Chapter9"></a><h2>Obsolete Functions</h2><pre></pre>
+<a name="Chapter10"></a><h2>Obsolete Functions</h2><pre></pre>
 
 <pre><b>#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
 #  define LZ4_DEPRECATED(message)   </b>/* disable deprecation warnings */<b>
diff --git a/lib/lz4.c b/lib/lz4.c
index 21892d3..f55e4e1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -517,15 +517,15 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab
     return LZ4_hash4(LZ4_read32(p), tableType);
 }
 
-static void LZ4_putIndexOnHash(U32 index, U32 h, void* tableBase, tableType_t const tableType)
+static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType)
 {
     switch (tableType)
     {
     default: /* fallthrough */
     case clearedTable: /* fallthrough */
     case byPtr: { /* illegal! */ assert(0); return; }
-    case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = index; return; }
-    case byU16: { U16* hashTable = (U16*) tableBase; assert(index < 65536); hashTable[h] = (U16)index; return; }
+    case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; }
+    case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; }
     }
 }
 
-- 
cgit v0.12


From 54ec83ce1f62014398b76a441d1ff1212dad9604 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Apr 2018 02:10:53 -0700
Subject: fixed potential ptrdiff_t overflow (32-bits mode)

Also removed pointer comparison, which should solve #485
---
 lib/lz4.c | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index f55e4e1..c48aca4 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1315,15 +1315,14 @@ void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dic
 }
 
 
-static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
+static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize)
 {
-    if ((LZ4_dict->currentOffset > 0x80000000) ||
-        ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) {   /* address space overflow */
+    if (LZ4_dict->currentOffset + nextSize > 0x80000000) {   /* potential ptrdiff_t overflow (32-bits mode) */
         /* rescale hash table */
         U32 const delta = LZ4_dict->currentOffset - 64 KB;
         const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
         int i;
-        DEBUGLOG(4, "LZ4_renormDictT %p", LZ4_dict);
+        DEBUGLOG(4, "LZ4_renormDictT");
         for (i=0; i<LZ4_HASH_SIZE_U32; i++) {
             if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0;
             else LZ4_dict->hashTable[i] -= delta;
@@ -1341,10 +1340,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
     LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
     const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
-    const BYTE* smallest = (const BYTE*) source;
     if (streamPtr->initCheck) return 0;   /* Uninitialized structure detected */
-    if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd;
-    LZ4_renormDictT(streamPtr, smallest);
+    LZ4_renormDictT(streamPtr, inputSize);   /* avoid index overflow */
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
 
     /* Check overlapping input/dictionary space */
@@ -1377,7 +1374,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
             if (inputSize > 4 KB) {
                 /* For compressing large blobs, it is faster to pay the setup
                  * cost to copy the dictionary's tables into the active context,
-                 * so that the compression loop is only looking in one table.
+                 * so that the compression loop is only looking into one table.
                  */
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
@@ -1398,8 +1395,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 }
 
 
-/* Hidden debug function, to force external dictionary mode */
-int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize)
+/* Hidden debug function, to force-test external dictionary mode */
+int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize)
 {
     LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse;
     int result;
@@ -1407,16 +1404,16 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
 
     const BYTE* smallest = dictEnd;
     if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
-    LZ4_renormDictT(streamPtr, smallest);
+    LZ4_renormDictT(streamPtr, srcSize);
 
     if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
     } else {
-        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
     }
 
     streamPtr->dictionary = (const BYTE*)source;
-    streamPtr->dictSize = (U32)inputSize;
+    streamPtr->dictSize = (U32)srcSize;
 
     return result;
 }
-- 
cgit v0.12


From c40bac31d3886c2f69d0364823b6d6aaa972ee5b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Apr 2018 02:26:14 -0700
Subject: added comment on variables required after _next_match

---
 lib/lz4.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index c48aca4..54d037c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -787,6 +787,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         }
 
 _next_match:
+        /* at this stage, the following variables must be correctly set :
+         * - ip : at start of LZ operation
+         * - match : at start of previous pattern occurence; can be within current prefix, or within extDict
+         * - offset : if maybe_ext_memSegment==1 (constant)
+         * - lowLimit : must be == dictionary to mean "match is within extDict"; must be == source otherwise
+         * - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written
+         */
+
         /* Encode Offset */
         if (maybe_ext_memSegment) {   /* static test */
             assert(offset <= MAX_DISTANCE && offset > 0);
-- 
cgit v0.12


From d2bcfa31f525aaa11c2d248af0ba487791399c1f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Apr 2018 02:45:32 -0700
Subject: fixed minor unused variable warning

---
 lib/lz4.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 54d037c..e8f9831 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1408,10 +1408,7 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
 {
     LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse;
     int result;
-    const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
-    const BYTE* smallest = dictEnd;
-    if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
     LZ4_renormDictT(streamPtr, srcSize);
 
     if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-- 
cgit v0.12


From 6dd64e0776dccf5c19e8438625657abf0e6f3048 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 12 Apr 2018 18:23:01 -0400
Subject: Add Tests for LZ4_attach_dictionary and Friends

---
 tests/fuzzer.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c134fe3..98b3d9f 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -43,6 +43,7 @@
 #include <stdio.h>      /* fgets, sscanf */
 #include <string.h>     /* strcmp */
 #include <time.h>       /* clock_t, clock, CLOCKS_PER_SEC */
+#define LZ4_STATIC_LINKING_ONLY
 #define LZ4_HC_STATIC_LINKING_ONLY
 #include "lz4hc.h"
 #define XXH_STATIC_LINKING_ONLY
@@ -402,6 +403,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
 
+        /* Test compression using fast reset external state*/
+        FUZ_DISPLAYTEST;
+        ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed");
+
         /* Test compression */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
@@ -626,6 +632,78 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
                 FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
         }   }
 
+        /* Compress using external dictionary stream */
+        FUZ_DISPLAYTEST;
+        {
+            LZ4_stream_t LZ4_stream;
+
+            LZ4_loadDict(&LZ4dict, dict, dictSize);
+
+            LZ4_resetStream(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
+
+            FUZ_DISPLAYTEST;
+            LZ4_resetStream(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
+            FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
+
+            FUZ_DISPLAYTEST;
+            LZ4_resetStream(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
+
+            FUZ_DISPLAYTEST;
+            LZ4_resetStream_fast(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
+        }
+
+        /* Decompress with dictionary as external */
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize] = 0;
+        ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
+        FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
+        FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
+        crcCheck = XXH32(decodedBuffer, blockSize, 0);
+        if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize] = 0;
+        ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
+        FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
+        FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+        crcCheck = XXH32(decodedBuffer, blockSize, 0);
+        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize-1] = 0;
+        ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
+        FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
+        FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
+
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize-1] = 0;
+        ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
+        FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
+        FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+
+        FUZ_DISPLAYTEST;
+        {   U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
+            if ((U32)blockSize > missingBytes) {
+                decodedBuffer[blockSize-missingBytes] = 0;
+                ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
+                FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
+                FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
+        }   }
+
         /* Compress HC using External dictionary */
         FUZ_DISPLAYTEST;
         dict -= (FUZ_rand(&randState) & 7);    /* even bigger separation */
-- 
cgit v0.12


From 9f0f6b89bb27d4d6c5d12a7142fc330f3d1b3a10 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 12 Apr 2018 19:17:53 -0400
Subject: Further Test that ExtDictCtx Mode Produces the Exact Same Output

---
 tests/fuzzer.c | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 98b3d9f..db05738 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -303,9 +303,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         U32 testNb = 0;
         U32 randState = FUZ_rand(&coreRandState) ^ PRIME3;
         int const blockSize  = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1;
-        int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
+        int const blockStart = (FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
         int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
-        int const dictSize = MIN(dictSizeRand, blockStart);
+        int const dictSize = MIN(dictSizeRand, blockStart - 1);
         int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
         char* const block = ((char*)CNBuffer) + blockStart;
         const char* dict = block - dictSize;
@@ -636,6 +636,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_DISPLAYTEST;
         {
             LZ4_stream_t LZ4_stream;
+            int expectedSize;
+            U32 expectedCrc;
+
+            LZ4_loadDict(&LZ4dict, dict, dictSize);
+            expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+            FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
+            expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
 
             LZ4_loadDict(&LZ4dict, dict, dictSize);
 
@@ -644,6 +651,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
 
+            /* In the future, it might be desirable to let extDictCtx mode's
+             * output diverge from the output generated by regular extDict mode.
+             * Until that time, this comparison serves as a good regression
+             * test.
+             */
+            FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
+            FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
+
             FUZ_DISPLAYTEST;
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
@@ -656,6 +671,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
+            FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
+            FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
             FUZ_DISPLAYTEST;
             LZ4_resetStream_fast(&LZ4_stream);
@@ -663,6 +680,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
+            FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
+            FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
         }
 
         /* Decompress with dictionary as external */
-- 
cgit v0.12


From e9280647976ca468ab67ffbf3dd9c8e533fe23bf Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Apr 2018 15:11:28 -0700
Subject: fixed gcc performance regression

---
 lib/lz4.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index e8f9831..c2012f6 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -660,6 +660,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
+    assert(acceleration >= 1);
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
 
@@ -712,6 +713,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         } else {   /* byU32, byU16 */
 
             const BYTE* forwardIp = ip;
+            unsigned step = 1;
             unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
             do {
                 U32 const h = forwardH;
@@ -720,8 +722,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 assert(matchIndex <= current);
                 assert(forwardIp - base < (ptrdiff_t)(2 GB - 1));
                 ip = forwardIp;
-                assert(searchMatchNb >= (1<<LZ4_skipTrigger));
-                forwardIp += (searchMatchNb++ >> LZ4_skipTrigger);
+                forwardIp += step;
+                step = (searchMatchNb++ >> LZ4_skipTrigger);
 
                 if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
                 assert(ip < mflimitPlusOne);
-- 
cgit v0.12


From 4aff9b10b56e44b2ff6326f611c3659565812d80 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Apr 2018 16:14:28 -0700
Subject: fixed fuzzer tests

which were modified in parallel within branc `dev`
---
 tests/fuzzer.c | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 83ab50b..7721345 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -454,7 +454,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
 
         /* Test compression using fast reset external state*/
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed");
 
@@ -687,7 +687,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Compress using external dictionary stream */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         {
             LZ4_stream_t LZ4_stream;
             int expectedSize;
@@ -713,13 +713,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
             FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
             FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
 
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
@@ -728,7 +728,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
             FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             LZ4_resetStream_fast(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
@@ -739,36 +739,38 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
 
         /* Decompress with dictionary as external */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+        }
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        }
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         {   U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
             if ((U32)blockSize > missingBytes) {
                 decodedBuffer[blockSize-missingBytes] = 0;
-- 
cgit v0.12


From a3aeb34184e20d51616beccfcbbe7aade3cc3a64 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Apr 2018 16:54:03 -0700
Subject: fixed minor format warnings

---
 lib/lz4.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c2012f6..980a5fd 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -785,7 +785,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             /* Copy Literals */
             LZ4_wildCopy(op, anchor, op+litLength);
             op+=litLength;
-            DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, litLength, ip-(const BYTE*)source);
+            DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
         }
 
 _next_match:
@@ -801,7 +801,7 @@ _next_match:
         if (maybe_ext_memSegment) {   /* static test */
             assert(offset <= MAX_DISTANCE && offset > 0);
             LZ4_writeLE16(op, (U16)offset); op+=2;
-            DEBUGLOG(6, "                with offset=%u  (ext if > %zi)", offset, ip - (const BYTE*)source);
+            DEBUGLOG(6, "                with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
         } else  {
             assert(ip-match <= MAX_DISTANCE);
             LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
@@ -901,7 +901,7 @@ _next_match:
                 *token=0;
                 if (maybe_ext_memSegment)
                     offset = current - matchIndex;
-                DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, 0, ip-(const BYTE*)source);
+                DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
                 goto _next_match;
             }
         }
-- 
cgit v0.12


From 444211d2599a2be59e3f50418b46ec2431288e9a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Apr 2018 17:15:02 -0700
Subject: edited a few traces for debugging

---
 lib/lz4.c      | 14 +++++++-------
 tests/fuzzer.c |  6 +++---
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 980a5fd..dca4d69 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -588,12 +588,12 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
           || tableType == byPtr
           || inputSize >= 4 KB)
         {
-            DEBUGLOG(4, "Resetting table in %p", cctx);
+            DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx);
             MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
             cctx->currentOffset = 0;
             cctx->tableType = clearedTable;
         } else {
-            DEBUGLOG(4, "Re-use hash table (no reset)");
+            DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)");
         }
     }
 
@@ -799,13 +799,13 @@ _next_match:
 
         /* Encode Offset */
         if (maybe_ext_memSegment) {   /* static test */
+            DEBUGLOG(6, "             with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
             assert(offset <= MAX_DISTANCE && offset > 0);
             LZ4_writeLE16(op, (U16)offset); op+=2;
-            DEBUGLOG(6, "                with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
         } else  {
+            DEBUGLOG(6, "             with offset=%u  (same segment)", (U32)(ip - match));
             assert(ip-match <= MAX_DISTANCE);
             LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
-            DEBUGLOG(6, "                with offset=%u  (same segment)", (U32)(ip - match));
         }
 
         /* Encode MatchLength */
@@ -823,11 +823,11 @@ _next_match:
                     matchCode += more;
                     ip += more;
                 }
-                DEBUGLOG(6, "                with matchLength=%u starting in extDict", matchCode+MINMATCH);
+                DEBUGLOG(6, "             with matchLength=%u starting in extDict", matchCode+MINMATCH);
             } else {
                 matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
                 ip += MINMATCH + matchCode;
-                DEBUGLOG(6, "                with matchLength=%u", matchCode+MINMATCH);
+                DEBUGLOG(6, "             with matchLength=%u", matchCode+MINMATCH);
             }
 
             if ( outputLimited &&    /* Check output buffer overflow */
@@ -1259,7 +1259,7 @@ LZ4_stream_t* LZ4_createStream(void)
 
 void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
-    DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
+    DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
 }
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 7721345..244cc4f 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -687,19 +687,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Compress using external dictionary stream */
-        FUZ_DISPLAYTEST();
         {
             LZ4_stream_t LZ4_stream;
             int expectedSize;
             U32 expectedCrc;
 
+            FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_loadDict()");
             LZ4_loadDict(&LZ4dict, dict, dictSize);
             expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
             FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
             expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
 
+            FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()");
             LZ4_loadDict(&LZ4dict, dict, dictSize);
-
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
@@ -713,7 +713,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
             FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
-            FUZ_DISPLAYTEST();
+            FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short");
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
-- 
cgit v0.12


From da3b5ba6f0014564b7312511e441067ba9429733 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Apr 2018 23:59:42 -0700
Subject: fixed dictCtx compression

---
 lib/lz4.c | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index dca4d69..0590de4 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -602,6 +602,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * currentOffset == 0 is faster still, so we preserve that case.
      */
     if (cctx->currentOffset != 0 && tableType == byU32) {
+        DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset");
         cctx->currentOffset += 64 KB;
     }
 
@@ -636,8 +637,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
         dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
+    const U32 dictDelta = usingDictCtx ? startIndex - dictCtx->currentOffset : 0;   /* make indexes in dictCtx comparable with index in current context */
 
-    int const maybe_ext_memSegment = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
+    int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
     U32 const prefixIdxLimit = startIndex - dictSize;   /* used when dictDirective == dictSmall */
     const BYTE* const dictEnd = dictionary + dictSize;
     const BYTE* anchor = (const BYTE*) source;
@@ -648,7 +650,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* the dictCtx currentOffset is indexed on the start of the dictionary,
      * while a dictionary in the current context precedes the currentOffset */
     const BYTE* dictBase = dictDirective == usingDictCtx ?
-        dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */
+        dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? Yes if the dictionary context is not reset */
         dictionary + dictSize - startIndex;
 
     BYTE* op = (BYTE*) dest;
@@ -657,6 +659,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     U32 offset = 0;
     U32 forwardH;
 
+    DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType);
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
@@ -731,8 +734,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 if (dictDirective == usingDictCtx) {
                     if (matchIndex < startIndex) {
                         /* there was no match, try the dictionary */
+                        assert(tableType == byU32);
                         matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                         match = dictBase + matchIndex;
+                        matchIndex += dictDelta;   /* make dictCtx index comparable with current context */
                         lowLimit = dictionary;
                     } else {
                         match = base + matchIndex;
@@ -758,7 +763,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);     /* too_far presumed impossible with byU16 */
 
                 if (LZ4_read32(match) == LZ4_read32(ip)) {
-                    if (maybe_ext_memSegment) offset = current - matchIndex;
+                    if (maybe_extMem) offset = current - matchIndex;
                     break;   /* match found */
                 }
 
@@ -798,7 +803,7 @@ _next_match:
          */
 
         /* Encode Offset */
-        if (maybe_ext_memSegment) {   /* static test */
+        if (maybe_extMem) {   /* static test */
             DEBUGLOG(6, "             with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
             assert(offset <= MAX_DISTANCE && offset > 0);
             LZ4_writeLE16(op, (U16)offset); op+=2;
@@ -878,6 +883,7 @@ _next_match:
                     matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                     match = dictBase + matchIndex;
                     lowLimit = dictionary;   /* required for match length counter */
+                    matchIndex += dictDelta;
                 } else {
                     match = base + matchIndex;
                     lowLimit = (const BYTE*)source;  /* required for match length counter */
@@ -899,8 +905,7 @@ _next_match:
               && (LZ4_read32(match) == LZ4_read32(ip)) ) {
                 token=op++;
                 *token=0;
-                if (maybe_ext_memSegment)
-                    offset = current - matchIndex;
+                if (maybe_extMem) offset = current - matchIndex;
                 DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
                 goto _next_match;
             }
@@ -1285,7 +1290,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
-    DEBUGLOG(4, "LZ4_loadDict (%p into %p)", dictionary, LZ4_dict);
+    DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict);
 
     LZ4_prepareTable(dict, 0, tableType);
 
-- 
cgit v0.12


From aedc44780468c11d90114bc1d6124af545bfc652 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 17 Apr 2018 14:01:44 -0400
Subject: Always Bump Offset by 64 KB in LZ4_loadDict()

This actually ensures the guarantee referred to in the comment in
LZ4_compress_fast_continue().
---
 lib/lz4.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4b0efb1..0ce05da 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1173,11 +1173,18 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 
     LZ4_prepareTable(dict, 0, tableType);
 
+    /* We always increment the offset by 64 KB, since, if the dict is longer,
+     * we truncate it to the last 64k, and if it's shorter, we still want to
+     * advance by a whole window length so we can provide the guarantee that
+     * there are only valid offsets in the window, which allows an optimization
+     * in LZ4_compress_fast_continue() where it uses noDictIssue even when the
+     * dictionary isn't a full 64k. */
+
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
-    base = p - dict->currentOffset;
+    base = dictEnd - 64 KB - dict->currentOffset;
     dict->dictionary = p;
     dict->dictSize = (U32)(dictEnd - p);
-    dict->currentOffset += dict->dictSize;
+    dict->currentOffset += 64 KB;
     dict->tableType = tableType;
 
     if (dictSize < (int)HASH_UNIT) {
-- 
cgit v0.12


From 152064218361e5762fd67b5de425707fdc47095b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 17 Apr 2018 15:29:17 -0700
Subject: fix matchIndex overflow

can happen with dictCtx
---
 lib/lz4.c | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index b426545..c799596 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -758,9 +758,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
 
-                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;    /* match outside of valid area */
-                if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue;  /* too far */
-                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);     /* too_far presumed impossible with byU16 */
+                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;     /* match outside of valid area */
+                if ((tableType != byU16) && (current - matchIndex > MAX_DISTANCE)) continue; /* too far - note: works even if matchIndex overflows */
+                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);      /* too_far presumed impossible with byU16 */
 
                 if (LZ4_read32(match) == LZ4_read32(ip)) {
                     if (maybe_extMem) offset = current - matchIndex;
@@ -861,7 +861,6 @@ _next_match:
         /* Fill table */
         LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);
 
-#if 1
         /* Test next position */
         if (tableType == byPtr) {
 
@@ -901,7 +900,7 @@ _next_match:
             }
             LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
             if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
-              && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current))
+              && ((tableType==byU16) ? 1 : (current - matchIndex <= MAX_DISTANCE))
               && (LZ4_read32(match) == LZ4_read32(ip)) ) {
                 token=op++;
                 *token=0;
@@ -914,13 +913,6 @@ _next_match:
         /* Prepare next loop */
         forwardH = LZ4_hashPosition(++ip, tableType);
 
-#else
-
-        /* Prepare next loop */
-        forwardH = LZ4_hashPosition(ip, tableType);
-
-#endif
-
     }
 
 _last_literals:
-- 
cgit v0.12


From 88cca1723e76c8f5031954ba07b28447c0cb55d8 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 17 Apr 2018 16:18:37 -0700
Subject: fix dictDelta setting error

wrong test
---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c799596..e7553ed 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -637,7 +637,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
         dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
-    const U32 dictDelta = usingDictCtx ? startIndex - dictCtx->currentOffset : 0;   /* make indexes in dictCtx comparable with index in current context */
+    const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0;   /* make indexes in dictCtx comparable with index in current context */
 
     int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
     U32 const prefixIdxLimit = startIndex - dictSize;   /* used when dictDirective == dictSmall */
-- 
cgit v0.12


From 5ad4599c5ad18d2408a6ccc545c45a36b99f0c6f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 17 Apr 2018 16:47:56 -0700
Subject: fixed LZ4_compress_fast_extState_fastReset() in 32-bit mode

---
 lib/lz4.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index e7553ed..33aa5c7 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -983,7 +983,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
                 return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
             LZ4_prepareTable(ctx, srcSize, tableType);
             return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
@@ -997,7 +997,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
                 return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
             LZ4_prepareTable(ctx, srcSize, tableType);
             return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
-- 
cgit v0.12


From ea6ed46fc273a7b10870caeb515622737eb1c572 Mon Sep 17 00:00:00 2001
From: Dmitrii Rodionov <d.rodionov@samsung.com>
Date: Wed, 18 Apr 2018 12:20:56 +0300
Subject: Wrap likely/unlikely macroses with #ifndef

It prevent redefine error when project using lz4 has its own likely/unlikely
macroses.
---
 lib/lz4.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0ce05da..1e96931 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -147,8 +147,12 @@
 #  define expect(expr,value)    (expr)
 #endif
 
+#ifndef likely
 #define likely(expr)     expect((expr) != 0, 1)
+#endif
+#ifndef unlikely
 #define unlikely(expr)   expect((expr) != 0, 0)
+#endif
 
 
 /*-************************************
-- 
cgit v0.12


From 4785bd6a3508c6e10c5cee7126fd112a0ec44599 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 18 Apr 2018 16:49:27 -0700
Subject: minor length reduction of several large lines

---
 doc/lz4frame_manual.html | 50 ++++++++++++++++++-------------
 lib/Makefile             |  4 +++
 lib/lz4.c                | 76 +++++++++++++++++++++++++++++++++---------------
 lib/lz4frame.h           | 50 ++++++++++++++++++-------------
 4 files changed, 115 insertions(+), 65 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index f0f7f5a..459bac8 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -30,9 +30,9 @@
 
 <a name="Chapter3"></a><h2>Error management</h2><pre></pre>
 
-<pre><b>unsigned    LZ4F_isError(LZ4F_errorCode_t code);   </b>/**< tells if a `LZ4F_errorCode_t` function result is an error code */<b>
+<pre><b>unsigned    LZ4F_isError(LZ4F_errorCode_t code);   </b>/**< tells when a function result is an error code */<b>
 </b></pre><BR>
-<pre><b>const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   </b>/**< return error code string; useful for debugging */<b>
+<pre><b>const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   </b>/**< return error code string; for debugging */<b>
 </b></pre><BR>
 <a name="Chapter4"></a><h2>Frame compression types</h2><pre></pre>
 
@@ -74,13 +74,13 @@
 } LZ4F_frameType_t;
 </b></pre><BR>
 <pre><b>typedef struct {
-  LZ4F_blockSizeID_t     blockSizeID;          </b>/* max64KB, max256KB, max1MB, max4MB ; 0 == default */<b>
-  LZ4F_blockMode_t       blockMode;            </b>/* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */<b>
-  LZ4F_contentChecksum_t contentChecksumFlag;  </b>/* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default)  */<b>
-  LZ4F_frameType_t       frameType;            </b>/* read-only field : LZ4F_frame or LZ4F_skippableFrame */<b>
-  unsigned long long     contentSize;          </b>/* Size of uncompressed content ; 0 == unknown */<b>
-  unsigned               dictID;               </b>/* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */<b>
-  LZ4F_blockChecksum_t   blockChecksumFlag;    </b>/* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default)  */<b>
+  LZ4F_blockSizeID_t     blockSizeID;         </b>/* max64KB, max256KB, max1MB, max4MB; 0 == default */<b>
+  LZ4F_blockMode_t       blockMode;           </b>/* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */<b>
+  LZ4F_contentChecksum_t contentChecksumFlag; </b>/* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */<b>
+  LZ4F_frameType_t       frameType;           </b>/* read-only field : LZ4F_frame or LZ4F_skippableFrame */<b>
+  unsigned long long     contentSize;         </b>/* Size of uncompressed content ; 0 == unknown */<b>
+  unsigned               dictID;              </b>/* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */<b>
+  LZ4F_blockChecksum_t   blockChecksumFlag;   </b>/* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */<b>
 } LZ4F_frameInfo_t;
 </b><p>  makes it possible to set or read frame parameters.
   It's not required to set all fields, as long as the structure was initially memset() to zero.
@@ -89,7 +89,7 @@
 
 <pre><b>typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       </b>/* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */<b>
+  int      compressionLevel;       </b>/* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */<b>
   unsigned autoFlush;              </b>/* 1 == always flush, to reduce usage of internal buffers */<b>
   unsigned reserved[4];            </b>/* must be zero for forward compatibility */<b>
 } LZ4F_preferences_t;
@@ -165,34 +165,42 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  
 </p></pre><BR>
 
-<pre><b>size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
+<pre><b>size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
+                                       void* dstBuffer, size_t dstCapacity,
+                                 const void* srcBuffer, size_t srcSize,
+                                 const LZ4F_compressOptions_t* cOptPtr);
 </b><p>  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
-  An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
+  Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.
   This value is provided by LZ4F_compressBound().
   If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
-  LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
+  LZ4F_compressUpdate() doesn't guarantee error recovery.
+  When an error occurs, compression context must be freed or resized.
  `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
            or an error code if it fails (which can be tested using LZ4F_isError())
  
 </p></pre><BR>
 
-<pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+<pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx,
+                              void* dstBuffer, size_t dstCapacity,
+                        const LZ4F_compressOptions_t* cOptPtr);
 </b><p>  When data must be generated and sent immediately, without waiting for a block to be completely filled,
   it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
  `dstCapacity` must be large enough to ensure the operation will be successful.
  `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
- @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
+ @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
            or an error code if it fails (which can be tested using LZ4F_isError())
  
 </p></pre><BR>
 
-<pre><b>size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+<pre><b>size_t LZ4F_compressEnd(LZ4F_cctx* cctx,
+                                    void* dstBuffer, size_t dstCapacity,
+                              const LZ4F_compressOptions_t* cOptPtr);
 </b><p>  To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
   It will flush whatever data remained within `cctx` (like LZ4_flush())
   and properly finalize the frame, with an endMark and a checksum.
  `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
- @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
+ @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark),
            or an error code if it fails (which can be tested using LZ4F_isError())
   A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
  
@@ -201,7 +209,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 <a name="Chapter9"></a><h2>Decompression functions</h2><pre></pre>
 
 <pre><b>typedef struct {
-  unsigned stableDst;    </b>/* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */<b>
+  unsigned stableDst;    </b>/* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */<b>
   unsigned reserved[3];  </b>/* must be set to zero for forward compatibility */<b>
 } LZ4F_decompressOptions_t;
 </b></pre><BR>
@@ -212,7 +220,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
   The function provides a pointer to an allocated and initialized LZ4F_dctx object.
   The result is an errorCode, which can be tested using LZ4F_isError().
   dctx memory can be released using LZ4F_freeDecompressionContext();
-  The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+  Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.
   That is, it should be == 0 if decompression has been completed fully and correctly.
  
 </p></pre><BR>
@@ -252,8 +260,8 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
   The function will read up to *srcSizePtr bytes from srcBuffer,
   and decompress data into dstBuffer, of capacity *dstSizePtr.
 
-  The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
-  The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
+  The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
+  The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
 
   The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
   Unconsumed source data must be presented again in subsequent invocations.
diff --git a/lib/Makefile b/lib/Makefile
index dd33f50..7b31239 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -116,6 +116,10 @@ clean:
 #-----------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 
+.PHONY: listL120
+listL120:  # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility)
+	find . -type f -name '*.c' -o -name '*.h' | while read -r filename; do awk 'length > 120 {print FILENAME "(" FNR "): " $$0}' $$filename; done
+
 DESTDIR     ?=
 # directory variables : GNU conventions prefer lowercase
 # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
diff --git a/lib/lz4.c b/lib/lz4.c
index 33aa5c7..af3221c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -69,7 +69,9 @@
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef LZ4_FORCE_MEMORY_ACCESS   /* can be defined externally */
-#  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+#  if defined(__GNUC__) && \
+  ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \
+  || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define LZ4_FORCE_MEMORY_ACCESS 2
 #  elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__)
 #    define LZ4_FORCE_MEMORY_ACCESS 1
@@ -80,7 +82,7 @@
  * LZ4_FORCE_SW_BITCOUNT
  * Define this parameter if your target system or compiler does not support hardware bit count
  */
-#if defined(_MSC_VER) && defined(_WIN32_WCE)   /* Visual Studio for Windows CE does not support Hardware bit count */
+#if defined(_MSC_VER) && defined(_WIN32_WCE)   /* Visual Studio for WinCE doesn't support Hardware bit count */
 #  define LZ4_FORCE_SW_BITCOUNT
 #endif
 
@@ -318,7 +320,7 @@ static const int LZ4_minLength = (MFLIMIT+1);
 #  endif
 #endif
 
-#define LZ4_STATIC_ASSERT(c)   { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
+#define LZ4_STATIC_ASSERT(c)   { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use after variable declarations */
 
 #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
 #  include <stdio.h>
@@ -529,7 +531,9 @@ static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t cons
     }
 }
 
-static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
+static void LZ4_putPositionOnHash(const BYTE* p, U32 h,
+                                  void* tableBase, tableType_t const tableType,
+                            const BYTE* srcBase)
 {
     switch (tableType)
     {
@@ -555,8 +559,16 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_
 static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType)
 {
     LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2);
-    if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-2))); return hashTable[h]; }
-    if (tableType == byU16) { const U16* const hashTable = (const U16*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-1))); return hashTable[h]; }
+    if (tableType == byU32) {
+        const U32* const hashTable = (const U32*) tableBase;
+        assert(h < (1U << (LZ4_MEMORY_USAGE-2)));
+        return hashTable[h];
+    }
+    if (tableType == byU16) {
+        const U16* const hashTable = (const U16*) tableBase;
+        assert(h < (1U << (LZ4_MEMORY_USAGE-1)));
+        return hashTable[h];
+    }
     assert(0); return 0;  /* forbidden case */
 }
 
@@ -567,7 +579,9 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType
     { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
 }
 
-LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
+LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p,
+                                             const void* tableBase, tableType_t tableType,
+                                             const BYTE* srcBase)
 {
     U32 const h = LZ4_hashPosition(p, tableType);
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
@@ -650,7 +664,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* the dictCtx currentOffset is indexed on the start of the dictionary,
      * while a dictionary in the current context precedes the currentOffset */
     const BYTE* dictBase = dictDirective == usingDictCtx ?
-        dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? Yes if the dictionary context is not reset */
+        dictionary + dictSize - dictCtx->currentOffset :
         dictionary + dictSize - startIndex;
 
     BYTE* op = (BYTE*) dest;
@@ -667,7 +681,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
 
-    if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
+    if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;  /* Size too large (not within 64K limit) */
 
     /* Update context state */
     if (dictDirective == usingDictCtx) {
@@ -681,7 +695,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     cctx->currentOffset += (U32)inputSize;
     cctx->tableType = tableType;
 
-    if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
+    if (inputSize<LZ4_minLength) goto _last_literals;        /* Input too small, no compression (all literals) */
 
     /* First Byte */
     LZ4_putPosition(ip, cctx->hashTable, tableType, base);
@@ -790,7 +804,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             /* Copy Literals */
             LZ4_wildCopy(op, anchor, op+litLength);
             op+=litLength;
-            DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
+            DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",
+                        (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
         }
 
 _next_match:
@@ -905,7 +920,8 @@ _next_match:
                 token=op++;
                 *token=0;
                 if (maybe_extMem) offset = current - matchIndex;
-                DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
+                DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",
+                            (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
                 goto _next_match;
             }
         }
@@ -1649,19 +1665,25 @@ _output_error:
 LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
 {
-    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
+    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
+                                  endOnInputSize, full, 0, noDict,
+                                  (BYTE*)dest, NULL, 0);
 }
 
 LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
 {
-    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0);
+    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
+                                  endOnInputSize, partial, targetOutputSize,
+                                  noDict, (BYTE*)dest, NULL, 0);
 }
 
 LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
 {
-    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB);
+    return LZ4_decompress_generic(source, dest, 0, originalSize,
+                                  endOnOutputSize, full, 0, withPrefix64k,
+                                  (BYTE*)(dest - 64 KB), NULL, 64 KB);
 }
 
 
@@ -1803,12 +1825,18 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres
 *  Obsolete Functions
 ***************************************************/
 /* obsolete compression functions */
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
-int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
-int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
-int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
-int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
+int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) {
+    return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
+int LZ4_compress(const char* source, char* dest, int inputSize) {
+    return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
+int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) {
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
+int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) {
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) {
+    return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
+int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) {
+    return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
 
 /*
 These function names are deprecated and should no longer be used.
@@ -1816,8 +1844,10 @@ They are only provided here for compatibility with older user programs.
 - LZ4_uncompress is totally equivalent to LZ4_decompress_fast
 - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
 */
-int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
-int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+int LZ4_uncompress (const char* source, char* dest, int outputSize) {
+    return LZ4_decompress_fast(source, dest, outputSize); }
+int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) {
+    return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
 
 /* Obsolete Streaming functions */
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 9efaca0..e80b1a1 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -93,8 +93,8 @@ extern "C" {
  **************************************/
 typedef size_t LZ4F_errorCode_t;
 
-LZ4FLIB_API unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells if a `LZ4F_errorCode_t` function result is an error code */
-LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; useful for debugging */
+LZ4FLIB_API unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells when a function result is an error code */
+LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; for debugging */
 
 
 /*-************************************
@@ -162,13 +162,13 @@ typedef LZ4F_contentChecksum_t contentChecksum_t;
  *  It's not required to set all fields, as long as the structure was initially memset() to zero.
  *  For all fields, 0 sets it to default value */
 typedef struct {
-  LZ4F_blockSizeID_t     blockSizeID;          /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
-  LZ4F_blockMode_t       blockMode;            /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
-  LZ4F_contentChecksum_t contentChecksumFlag;  /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default)  */
-  LZ4F_frameType_t       frameType;            /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
-  unsigned long long     contentSize;          /* Size of uncompressed content ; 0 == unknown */
-  unsigned               dictID;               /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
-  LZ4F_blockChecksum_t   blockChecksumFlag;    /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default)  */
+  LZ4F_blockSizeID_t     blockSizeID;         /* max64KB, max256KB, max1MB, max4MB; 0 == default */
+  LZ4F_blockMode_t       blockMode;           /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */
+  LZ4F_contentChecksum_t contentChecksumFlag; /* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */
+  LZ4F_frameType_t       frameType;           /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
+  unsigned long long     contentSize;         /* Size of uncompressed content ; 0 == unknown */
+  unsigned               dictID;              /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */
+  LZ4F_blockChecksum_t   blockChecksumFlag;   /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */
 } LZ4F_frameInfo_t;
 
 /*! LZ4F_preferences_t :
@@ -177,7 +177,7 @@ typedef struct {
  *  All reserved fields must be set to zero. */
 typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
+  int      compressionLevel;       /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
   unsigned autoFlush;              /* 1 == always flush, to reduce usage of internal buffers */
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
@@ -264,36 +264,44 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t*
 
 /*! LZ4F_compressUpdate() :
  *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
- *  An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
+ *  Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.
  *  This value is provided by LZ4F_compressBound().
  *  If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
- *  LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
+ *  LZ4F_compressUpdate() doesn't guarantee error recovery.
+ *  When an error occurs, compression context must be freed or resized.
  * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  */
-LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
+                                       void* dstBuffer, size_t dstCapacity,
+                                 const void* srcBuffer, size_t srcSize,
+                                 const LZ4F_compressOptions_t* cOptPtr);
 
 /*! LZ4F_flush() :
  *  When data must be generated and sent immediately, without waiting for a block to be completely filled,
  *  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
  * `dstCapacity` must be large enough to ensure the operation will be successful.
  * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
- * @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
+ * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  */
-LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx,
+                              void* dstBuffer, size_t dstCapacity,
+                        const LZ4F_compressOptions_t* cOptPtr);
 
 /*! LZ4F_compressEnd() :
  *  To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
  *  It will flush whatever data remained within `cctx` (like LZ4_flush())
  *  and properly finalize the frame, with an endMark and a checksum.
  * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
- * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
+ * @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark),
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  *  A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
  */
-LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx,
+                                    void* dstBuffer, size_t dstCapacity,
+                              const LZ4F_compressOptions_t* cOptPtr);
 
 
 /*-*********************************
@@ -303,7 +311,7 @@ typedef struct LZ4F_dctx_s LZ4F_dctx;   /* incomplete type */
 typedef LZ4F_dctx* LZ4F_decompressionContext_t;   /* compatibility with previous API versions */
 
 typedef struct {
-  unsigned stableDst;    /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
+  unsigned stableDst;    /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */
   unsigned reserved[3];  /* must be set to zero for forward compatibility */
 } LZ4F_decompressOptions_t;
 
@@ -316,7 +324,7 @@ typedef struct {
  *  The function provides a pointer to an allocated and initialized LZ4F_dctx object.
  *  The result is an errorCode, which can be tested using LZ4F_isError().
  *  dctx memory can be released using LZ4F_freeDecompressionContext();
- *  The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+ *  Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.
  *  That is, it should be == 0 if decompression has been completed fully and correctly.
  */
 LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
@@ -357,8 +365,8 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
  *  The function will read up to *srcSizePtr bytes from srcBuffer,
  *  and decompress data into dstBuffer, of capacity *dstSizePtr.
  *
- *  The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
- *  The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
+ *  The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
+ *  The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
  *
  *  The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
  *  Unconsumed source data must be presented again in subsequent invocations.
-- 
cgit v0.12


From 95bde2a4ae4a92e984a5783ca1f09f44bf04fadb Mon Sep 17 00:00:00 2001
From: Baruch Siach <baruch@tkos.co.il>
Date: Thu, 19 Apr 2018 12:28:11 +0300
Subject: lib: allow to disable shared libraries

Just like BUILD_STATIC=no disables static libraries, BUILD_SHARED=no
disabled shared libraries. This is useful to support toolchains that do
not support shared libraries.
---
 lib/Makefile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/Makefile b/lib/Makefile
index dd33f50..976d57c 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -42,6 +42,7 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
 LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
 LIBVER  := $(shell echo $(LIBVER_SCRIPT))
 
+BUILD_SHARED:=yes
 BUILD_STATIC:=yes
 
 CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
@@ -92,6 +93,7 @@ ifeq ($(BUILD_STATIC),yes)  # can be disabled on command line
 endif
 
 $(LIBLZ4): $(SRCFILES)
+ifeq ($(BUILD_SHARED),yes)  # can be disabled on command line
 	@echo compiling dynamic library $(LIBVER)
 ifneq (,$(filter Windows%,$(OS)))
 	@$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
@@ -102,6 +104,7 @@ else
 	@ln -sf $@ liblz4.$(SHARED_EXT_MAJOR)
 	@ln -sf $@ liblz4.$(SHARED_EXT)
 endif
+endif
 
 liblz4: $(LIBLZ4)
 
@@ -159,9 +162,11 @@ ifeq ($(BUILD_STATIC),yes)
 	@$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
 	@$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
 endif
+ifeq ($(BUILD_SHARED),yes)
 	@$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
 	@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
 	@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
+endif
 	@echo Installing headers in $(INCLUDEDIR)
 	@$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
 	@$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
-- 
cgit v0.12


From 46058d71aa97cf29690f7638179104985c28d354 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 19 Apr 2018 10:50:40 -0700
Subject: modified indentation for consistency

---
 lib/lz4.c | 50 +++++++++++++++++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index af3221c..10674a3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1825,29 +1825,45 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres
 *  Obsolete Functions
 ***************************************************/
 /* obsolete compression functions */
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) {
-    return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
-int LZ4_compress(const char* source, char* dest, int inputSize) {
-    return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
-int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) {
-    return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
-int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) {
-    return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) {
-    return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
-int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) {
-    return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
+int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+    return LZ4_compress_default(source, dest, inputSize, maxOutputSize);
+}
+int LZ4_compress(const char* source, char* dest, int inputSize)
+{
+    return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize));
+}
+int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize)
+{
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1);
+}
+int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize)
+{
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1);
+}
+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity)
+{
+    return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1);
+}
+int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize)
+{
+    return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1);
+}
 
 /*
-These function names are deprecated and should no longer be used.
+These decompression functions are deprecated and should no longer be used.
 They are only provided here for compatibility with older user programs.
 - LZ4_uncompress is totally equivalent to LZ4_decompress_fast
 - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
 */
-int LZ4_uncompress (const char* source, char* dest, int outputSize) {
-    return LZ4_decompress_fast(source, dest, outputSize); }
-int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) {
-    return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+int LZ4_uncompress (const char* source, char* dest, int outputSize)
+{
+    return LZ4_decompress_fast(source, dest, outputSize);
+}
+int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize)
+{
+    return LZ4_decompress_safe(source, dest, isize, maxOutputSize);
+}
 
 /* Obsolete Streaming functions */
 
-- 
cgit v0.12


From b9836b2a030a3756b1f8cf1341875354c84dd207 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 14:01:57 -0400
Subject: Restore Framebench Tool

This reverts commit 70f14823a46719e81e808d9ed9df90f478bcfd3f.
---
 tests/Makefile     |   6 +-
 tests/framebench.c | 296 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 301 insertions(+), 1 deletion(-)
 create mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index 2b93c9f..f6a4ff3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -63,7 +63,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen
+all: fullbench fuzzer frametest datagen framebench
 
 all32: CFLAGS+=-m32
 all32: all
@@ -99,6 +99,9 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
+framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -110,6 +113,7 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
+        framebench$(EXT) \
         fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
diff --git a/tests/framebench.c b/tests/framebench.c
new file mode 100644
index 0000000..18b5ff3
--- /dev/null
+++ b/tests/framebench.c
@@ -0,0 +1,296 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <unistd.h>
+
+#define LZ4_STATIC_LINKING_ONLY
+#include "lz4.h"
+#include "lz4frame.h"
+#include "lz4frame_static.h"
+
+#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
+
+typedef struct {
+  size_t iter;
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  char *obuf;
+  size_t osize;
+  const char* ibuf;
+  size_t isize;
+  size_t num_ibuf;
+  const LZ4F_CDict* cdict;
+  LZ4F_preferences_t* prefs;
+  const LZ4F_compressOptions_t* options;
+} bench_params_t;
+
+size_t compress_frame(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressFrame_usingCDict(
+    cctx,
+    obuf,
+    osize,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    cdict,
+    prefs);
+  LZ4F_CHECK(oused);
+
+  return oused;
+}
+
+size_t compress_begin(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+  const LZ4F_compressOptions_t* options = p->options;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressUpdate(
+    cctx,
+    obuf,
+    oend - obuf,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    options);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
+  LZ4F_CHECK(oused);
+
+  return obuf - p->obuf;
+}
+
+size_t compress_default(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_extState(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4_stream_t *ctx = p->ctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+uint64_t bench(
+    size_t (*fun)(bench_params_t *),
+    uint64_t repetitions,
+    bench_params_t *params,
+    size_t *osizePtr
+) {
+  struct timespec start, end;
+  size_t i, osize = 0;
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
+
+  for (i = 0; i < repetitions; i++) {
+    size_t o;
+    params->iter = i;
+    o = fun(params);
+    if (!o) return 0;
+    osize += o;
+  }
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+
+  *osizePtr = osize / repetitions;
+  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+}
+
+int main(int argc, char *argv[]) {
+
+
+  struct stat st;
+  size_t bytes_read;
+
+  const char *dict_fn;
+  size_t dict_size;
+  char *dict_buf;
+  FILE *dict_file;
+
+  const char *in_fn;
+  size_t in_size;
+  size_t num_in_buf;
+  size_t cur_in_buf;
+  char *in_buf;
+  FILE *in_file;
+
+  size_t out_size;
+  char *out_buf;
+
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  LZ4F_CDict *cdict;
+  LZ4F_preferences_t prefs;
+  LZ4F_compressOptions_t options;
+
+  size_t out_used;
+
+  uint64_t time_taken;
+  uint64_t repetitions;
+
+  bench_params_t params;
+
+  if (argc != 3) return 1;
+  dict_fn = argv[1];
+  in_fn = argv[2];
+
+  if (stat(dict_fn, &st)) return 1;
+  dict_size = st.st_size;
+  dict_buf = (char *)malloc(dict_size);
+  if (!dict_buf) return 1;
+  dict_file = fopen(dict_fn, "r");
+  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
+  if (bytes_read != dict_size) return 1;
+
+  if (stat(in_fn, &st)) return 1;
+  in_size = st.st_size;
+  num_in_buf = 256 * 1024 * 1024 / in_size;
+  if (num_in_buf == 0) {
+    num_in_buf = 1;
+  }
+
+  in_buf = (char *)malloc(in_size * num_in_buf);
+  if (!in_buf) return 1;
+  in_file = fopen(in_fn, "r");
+  bytes_read = fread(in_buf, 1, in_size, in_file);
+  if (bytes_read != in_size) return 1;
+
+  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
+    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
+  }
+
+  if (in_size <= 1024) {
+    repetitions = 100000;
+  } else
+  if (in_size <= 16384) {
+    repetitions = 10000;
+  } else
+  if (in_size <= 131072) {
+    repetitions = 1000;
+  } else
+  if (in_size <= 1048576) {
+    repetitions = 100;
+  } else {
+    repetitions = 50;
+  }
+
+  memset(&prefs, 0, sizeof(prefs));
+  prefs.autoFlush = 1;
+  if (in_size < 64 * 1024)
+    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
+  prefs.frameInfo.contentSize = in_size;
+
+  memset(&options, 0, sizeof(options));
+  options.stableSrc = 1;
+
+  out_size = LZ4F_compressFrameBound(in_size, &prefs);
+  out_buf = (char *)malloc(out_size);
+  if (!out_buf) return 1;
+
+  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
+  if (cctx == NULL) return 1;
+
+  ctx = LZ4_createStream();
+  if (ctx == NULL) return 1;
+
+  cdict = LZ4F_createCDict(dict_buf, dict_size);
+  if (!cdict) return 1;
+
+  fprintf(stderr, "dict  size: %zd\n", dict_size);
+  fprintf(stderr, "input size: %zd\n", in_size);
+
+  params.ctx = ctx;
+  params.cctx = cctx;
+  params.obuf = out_buf;
+  params.osize = out_size;
+  params.ibuf = in_buf;
+  params.isize = in_size;
+  params.num_ibuf = num_in_buf;
+  params.cdict = NULL;
+  params.prefs = &prefs;
+  params.options = &options;
+
+  time_taken = bench(compress_default, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_extState, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  params.cdict = cdict;
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  return 0;
+}
\ No newline at end of file
-- 
cgit v0.12


From 0bc13ab69c603346ff64050f7e3d3a1b3a5765a4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 15:00:59 -0400
Subject: Add HC Calls to Framebench

---
 tests/framebench.c | 99 ++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 82 insertions(+), 17 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 18b5ff3..e2bcf40 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -10,6 +10,7 @@
 
 #define LZ4_STATIC_LINKING_ONLY
 #include "lz4.h"
+#include "lz4hc.h"
 #include "lz4frame.h"
 #include "lz4frame_static.h"
 
@@ -18,12 +19,14 @@
 typedef struct {
   size_t iter;
   LZ4_stream_t *ctx;
+  LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
   char *obuf;
   size_t osize;
   const char* ibuf;
   size_t isize;
   size_t num_ibuf;
+  int clevel;
   const LZ4F_CDict* cdict;
   LZ4F_preferences_t* prefs;
   const LZ4F_compressOptions_t* options;
@@ -117,11 +120,49 @@ size_t compress_extState(bench_params_t *p) {
   const char* ibuf = p->ibuf;
   size_t isize = p->isize;
   size_t num_ibuf = p->num_ibuf;
+  int clevel = p->clevel;
 
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_hc(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  int clevel = p->clevel;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_HC(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_hc_extState(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4_streamHC_t *hcctx = p->hcctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  int clevel = p->clevel;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_HC_extStateHC(hcctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -174,6 +215,7 @@ int main(int argc, char *argv[]) {
   char *out_buf;
 
   LZ4_stream_t *ctx;
+  LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
   LZ4F_CDict *cdict;
   LZ4F_preferences_t prefs;
@@ -216,18 +258,18 @@ int main(int argc, char *argv[]) {
   }
 
   if (in_size <= 1024) {
-    repetitions = 100000;
+    repetitions = 10000;
   } else
   if (in_size <= 16384) {
-    repetitions = 10000;
+    repetitions = 1000;
   } else
   if (in_size <= 131072) {
-    repetitions = 1000;
+    repetitions = 100;
   } else
   if (in_size <= 1048576) {
-    repetitions = 100;
+    repetitions = 10;
   } else {
-    repetitions = 50;
+    repetitions = 5;
   }
 
   memset(&prefs, 0, sizeof(prefs));
@@ -249,6 +291,9 @@ int main(int argc, char *argv[]) {
   ctx = LZ4_createStream();
   if (ctx == NULL) return 1;
 
+  hcctx = LZ4_createStreamHC();
+  if (hcctx == NULL) return 1;
+
   cdict = LZ4F_createCDict(dict_buf, dict_size);
   if (!cdict) return 1;
 
@@ -256,41 +301,61 @@ int main(int argc, char *argv[]) {
   fprintf(stderr, "input size: %zd\n", in_size);
 
   params.ctx = ctx;
+  params.hcctx = hcctx;
   params.cctx = cctx;
   params.obuf = out_buf;
   params.osize = out_size;
   params.ibuf = in_buf;
   params.isize = in_size;
   params.num_ibuf = num_in_buf;
+  params.clevel = 1;
   params.cdict = NULL;
   params.prefs = &prefs;
   params.options = &options;
 
   time_taken = bench(compress_default, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4_compress_default            : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_extState, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4_compress_fast_extState      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_frame, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4F_compressFrame              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_begin, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4F_compressBegin              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = cdict;
 
   time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  params.cdict = NULL;
+  params.clevel = LZ4HC_CLEVEL_MIN;
+  params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN;
+
+  time_taken = bench(compress_hc, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4_compress_HC                 : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_hc_extState, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4_compress_HC_extStateHC      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressFrame_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressBegin_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  params.cdict = cdict;
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   return 0;
-}
\ No newline at end of file
+}
-- 
cgit v0.12


From ffb2d8b0ed9615694c156ba1a3f51e9e789d3c4e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 22 Mar 2018 16:26:44 -0400
Subject: Check Compressed Buffer is Correct in Frame Bench

---
 tests/framebench.c | 79 ++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 65 insertions(+), 14 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index e2bcf40..a802ce0 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -21,11 +21,16 @@ typedef struct {
   LZ4_stream_t *ctx;
   LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
+  LZ4F_dctx *dctx;
+  const char *dictbuf;
+  size_t dictsize;
   char *obuf;
   size_t osize;
   const char* ibuf;
   size_t isize;
   size_t num_ibuf;
+  char *checkbuf;
+  size_t checksize;
   int clevel;
   const LZ4F_CDict* cdict;
   LZ4F_preferences_t* prefs;
@@ -168,19 +173,46 @@ size_t compress_hc_extState(bench_params_t *p) {
   return obuf - p->obuf;
 }
 
+size_t check_lz4(bench_params_t *p, size_t csize) {
+  (void)csize;
+  memset(p->checkbuf, 0xFF, p->checksize);
+  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize, p->dictbuf, p->dictsize)
+      && !memcmp(p->ibuf, p->checkbuf, p->isize);
+}
+
+size_t check_lz4f(bench_params_t *p, size_t csize) {
+  size_t cp = 0;
+  size_t dp = 0;
+  size_t dsize = p->checksize;
+  size_t cleft = csize;
+  size_t dleft = dsize;
+  size_t ret;
+  memset(p->checkbuf, 0xFF, p->checksize);
+  LZ4F_resetDecompressionContext(p->dctx);
+  do {
+    ret = LZ4F_decompress_usingDict(p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft, p->dictbuf, p->dictsize, NULL);
+    cp += cleft;
+    dp += dleft;
+    cleft = csize - cp;
+    dleft = dsize - dp;
+    if (LZ4F_isError(ret)) return 0;
+  } while (cleft);
+  return !memcmp(p->ibuf, p->checkbuf, p->isize);
+}
+
 uint64_t bench(
     size_t (*fun)(bench_params_t *),
+    size_t (*checkfun)(bench_params_t *, size_t),
     uint64_t repetitions,
     bench_params_t *params,
     size_t *osizePtr
 ) {
   struct timespec start, end;
-  size_t i, osize = 0;
+  size_t i, osize = 0, o = 0;
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
 
   for (i = 0; i < repetitions; i++) {
-    size_t o;
     params->iter = i;
     o = fun(params);
     if (!o) return 0;
@@ -189,6 +221,9 @@ uint64_t bench(
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
 
+  o = checkfun(params, o);
+  if (!o) return 0;
+
   *osizePtr = osize / repetitions;
   return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
 }
@@ -214,9 +249,13 @@ int main(int argc, char *argv[]) {
   size_t out_size;
   char *out_buf;
 
+  size_t check_size;
+  char *check_buf;
+
   LZ4_stream_t *ctx;
   LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
+  LZ4F_dctx *dctx;
   LZ4F_CDict *cdict;
   LZ4F_preferences_t prefs;
   LZ4F_compressOptions_t options;
@@ -257,6 +296,10 @@ int main(int argc, char *argv[]) {
     memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
   }
 
+  check_size = in_size;
+  check_buf = (char *)malloc(check_size);
+  if (!check_buf) return 1;
+
   if (in_size <= 1024) {
     repetitions = 10000;
   } else
@@ -288,6 +331,9 @@ int main(int argc, char *argv[]) {
   if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
   if (cctx == NULL) return 1;
 
+  if (LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) return 1;
+  if (cctx == NULL) return 1;
+
   ctx = LZ4_createStream();
   if (ctx == NULL) return 1;
 
@@ -303,58 +349,63 @@ int main(int argc, char *argv[]) {
   params.ctx = ctx;
   params.hcctx = hcctx;
   params.cctx = cctx;
+  params.dctx = dctx;
+  params.dictbuf = dict_buf;
+  params.dictsize = dict_size;
   params.obuf = out_buf;
   params.osize = out_size;
   params.ibuf = in_buf;
   params.isize = in_size;
   params.num_ibuf = num_in_buf;
+  params.checkbuf = check_buf;
+  params.checksize = check_size;
   params.clevel = 1;
   params.cdict = NULL;
   params.prefs = &prefs;
   params.options = &options;
 
-  time_taken = bench(compress_default, repetitions, &params, &out_used);
+  time_taken = bench(compress_default, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_default            : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_extState, repetitions, &params, &out_used);
+  time_taken = bench(compress_extState, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_fast_extState      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = cdict;
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = NULL;
   params.clevel = LZ4HC_CLEVEL_MIN;
   params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN;
 
-  time_taken = bench(compress_hc, repetitions, &params, &out_used);
+  time_taken = bench(compress_hc, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_HC                 : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_hc_extState, repetitions, &params, &out_used);
+  time_taken = bench(compress_hc_extState, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_HC_extStateHC      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = cdict;
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   return 0;
-- 
cgit v0.12


From d4ee7554fc79fecd5db83913536f5c856765bbed Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 16:19:30 -0400
Subject: Print More Detailed Results Inside bench(), Add Compression Levels

---
 tests/framebench.c | 103 ++++++++++++++++++++++++++---------------------------
 1 file changed, 51 insertions(+), 52 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index a802ce0..6d99b1e 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -111,7 +111,9 @@ size_t compress_default(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
+  oused = LZ4_compress_default(
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -130,7 +132,10 @@ size_t compress_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  oused = LZ4_compress_fast_extState_fastReset(
+      ctx,
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -148,7 +153,9 @@ size_t compress_hc(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_HC(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  oused = LZ4_compress_HC(
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -167,7 +174,10 @@ size_t compress_hc_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_HC_extStateHC(hcctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  oused = LZ4_compress_HC_extStateHC(
+      hcctx,
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -176,7 +186,8 @@ size_t compress_hc_extState(bench_params_t *p) {
 size_t check_lz4(bench_params_t *p, size_t csize) {
   (void)csize;
   memset(p->checkbuf, 0xFF, p->checksize);
-  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize, p->dictbuf, p->dictsize)
+  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize,
+                                       p->dictbuf, p->dictsize)
       && !memcmp(p->ibuf, p->checkbuf, p->isize);
 }
 
@@ -190,7 +201,9 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
   memset(p->checkbuf, 0xFF, p->checksize);
   LZ4F_resetDecompressionContext(p->dctx);
   do {
-    ret = LZ4F_decompress_usingDict(p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft, p->dictbuf, p->dictsize, NULL);
+    ret = LZ4F_decompress_usingDict(
+        p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft,
+        p->dictbuf, p->dictsize, NULL);
     cp += cleft;
     dp += dleft;
     cleft = csize - cp;
@@ -200,15 +213,17 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
   return !memcmp(p->ibuf, p->checkbuf, p->isize);
 }
 
+
 uint64_t bench(
+    char *bench_name,
     size_t (*fun)(bench_params_t *),
     size_t (*checkfun)(bench_params_t *, size_t),
     uint64_t repetitions,
-    bench_params_t *params,
-    size_t *osizePtr
+    bench_params_t *params
 ) {
   struct timespec start, end;
   size_t i, osize = 0, o = 0;
+  size_t time_taken;
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
 
@@ -224,8 +239,19 @@ uint64_t bench(
   o = checkfun(params, o);
   if (!o) return 0;
 
-  *osizePtr = osize / repetitions;
-  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+  time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
+               (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+
+  fprintf(
+      stderr,
+      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %12ld ns, %9ld ns/iter, %7.2lf MB/s\n",
+      bench_name, params->clevel,
+      params->isize, osize / repetitions,
+      repetitions, time_taken, time_taken / repetitions,
+      ((double) 1000 * params->isize * repetitions) / time_taken
+  );
+
+  return time_taken;
 }
 
 int main(int argc, char *argv[]) {
@@ -260,9 +286,8 @@ int main(int argc, char *argv[]) {
   LZ4F_preferences_t prefs;
   LZ4F_compressOptions_t options;
 
-  size_t out_used;
+  int clevels[] = {1, 2, 3, 6, 9, 10, 12};
 
-  uint64_t time_taken;
   uint64_t repetitions;
 
   bench_params_t params;
@@ -364,49 +389,23 @@ int main(int argc, char *argv[]) {
   params.prefs = &prefs;
   params.options = &options;
 
-  time_taken = bench(compress_default, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_default            : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_extState, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_fast_extState      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  params.cdict = cdict;
-
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  for (unsigned int clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
+    params.clevel = clevels[clevelidx];
+    params.prefs->compressionLevel = clevels[clevelidx];
+    params.cdict = NULL;
 
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+    bench("LZ4_compress_default"         , compress_default    , check_lz4 , repetitions, &params);
+    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , repetitions, &params);
+    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , repetitions, &params);
+    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , repetitions, &params);
+    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, repetitions, &params);
+    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, repetitions, &params);
 
-  params.cdict = NULL;
-  params.clevel = LZ4HC_CLEVEL_MIN;
-  params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN;
-
-  time_taken = bench(compress_hc, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_HC                 : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_hc_extState, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_HC_extStateHC      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+    params.cdict = cdict;
 
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  params.cdict = cdict;
-
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, repetitions, &params);
+    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, repetitions, &params);
+  }
 
   return 0;
 }
-- 
cgit v0.12


From a9a62321ffd077d1c4694b2762e84b60b2396c2d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 16:36:35 -0400
Subject: Auto-Calculate Appropriate Repetition Count

---
 tests/framebench.c | 67 +++++++++++++++++++++++-------------------------------
 1 file changed, 29 insertions(+), 38 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 6d99b1e..d5d2268 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -218,33 +218,41 @@ uint64_t bench(
     char *bench_name,
     size_t (*fun)(bench_params_t *),
     size_t (*checkfun)(bench_params_t *, size_t),
-    uint64_t repetitions,
     bench_params_t *params
 ) {
   struct timespec start, end;
   size_t i, osize = 0, o = 0;
-  size_t time_taken;
+  size_t time_taken = 0;
+  uint64_t total_repetitions = 0;
+  uint64_t repetitions = 2;
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
 
-  for (i = 0; i < repetitions; i++) {
-    params->iter = i;
-    o = fun(params);
-    if (!o) return 0;
-    osize += o;
-  }
+  while (time_taken < 25 * 1000 * 1000) { // benchmark over at least 1ms
+    if (total_repetitions) {
+      repetitions = total_repetitions; // double previous
+    }
+
+    for (i = 0; i < repetitions; i++) {
+      params->iter = i;
+      o = fun(params);
+      if (!o) return 0;
+      osize += o;
+    }
 
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+    if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+
+    time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
+                 (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+    total_repetitions += repetitions;
+  }
 
   o = checkfun(params, o);
   if (!o) return 0;
 
-  time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
-               (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
-
   fprintf(
       stderr,
-      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %12ld ns, %9ld ns/iter, %7.2lf MB/s\n",
+      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
       bench_name, params->clevel,
       params->isize, osize / repetitions,
       repetitions, time_taken, time_taken / repetitions,
@@ -288,8 +296,6 @@ int main(int argc, char *argv[]) {
 
   int clevels[] = {1, 2, 3, 6, 9, 10, 12};
 
-  uint64_t repetitions;
-
   bench_params_t params;
 
   if (argc != 3) return 1;
@@ -325,21 +331,6 @@ int main(int argc, char *argv[]) {
   check_buf = (char *)malloc(check_size);
   if (!check_buf) return 1;
 
-  if (in_size <= 1024) {
-    repetitions = 10000;
-  } else
-  if (in_size <= 16384) {
-    repetitions = 1000;
-  } else
-  if (in_size <= 131072) {
-    repetitions = 100;
-  } else
-  if (in_size <= 1048576) {
-    repetitions = 10;
-  } else {
-    repetitions = 5;
-  }
-
   memset(&prefs, 0, sizeof(prefs));
   prefs.autoFlush = 1;
   if (in_size < 64 * 1024)
@@ -394,17 +385,17 @@ int main(int argc, char *argv[]) {
     params.prefs->compressionLevel = clevels[clevelidx];
     params.cdict = NULL;
 
-    bench("LZ4_compress_default"         , compress_default    , check_lz4 , repetitions, &params);
-    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , repetitions, &params);
-    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , repetitions, &params);
-    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , repetitions, &params);
-    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, repetitions, &params);
-    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, repetitions, &params);
+    bench("LZ4_compress_default"         , compress_default    , check_lz4 , &params);
+    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , &params);
+    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , &params);
+    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , &params);
+    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, &params);
+    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, &params);
 
     params.cdict = cdict;
 
-    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, repetitions, &params);
-    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, repetitions, &params);
+    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, &params);
+    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, &params);
   }
 
   return 0;
-- 
cgit v0.12


From 09df7a05f9eb4471a5af7d6ca1bb37bea36ccf91 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 17:16:23 -0400
Subject: Add Run Name to Frame Bench Output

---
 tests/framebench.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index d5d2268..185d186 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -17,6 +17,7 @@
 #define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
 
 typedef struct {
+  const char *run_name;
   size_t iter;
   LZ4_stream_t *ctx;
   LZ4_streamHC_t *hcctx;
@@ -252,8 +253,8 @@ uint64_t bench(
 
   fprintf(
       stderr,
-      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
-      bench_name, params->clevel,
+      "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
+      params->run_name, bench_name, params->clevel,
       params->isize, osize / repetitions,
       repetitions, time_taken, time_taken / repetitions,
       ((double) 1000 * params->isize * repetitions) / time_taken
@@ -263,7 +264,7 @@ uint64_t bench(
 }
 
 int main(int argc, char *argv[]) {
-
+  char *run_name;
 
   struct stat st;
   size_t bytes_read;
@@ -298,9 +299,10 @@ int main(int argc, char *argv[]) {
 
   bench_params_t params;
 
-  if (argc != 3) return 1;
-  dict_fn = argv[1];
-  in_fn = argv[2];
+  if (argc != 4) return 1;
+  run_name = argv[1];
+  dict_fn = argv[2];
+  in_fn = argv[3];
 
   if (stat(dict_fn, &st)) return 1;
   dict_size = st.st_size;
@@ -362,6 +364,7 @@ int main(int argc, char *argv[]) {
   fprintf(stderr, "dict  size: %zd\n", dict_size);
   fprintf(stderr, "input size: %zd\n", in_size);
 
+  params.run_name = run_name;
   params.ctx = ctx;
   params.hcctx = hcctx;
   params.cctx = cctx;
-- 
cgit v0.12


From 66f0c29aa4d81586a4a621a99c763ab17028c270 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 13:12:28 -0400
Subject: Fix Framebench Statistics

---
 tests/framebench.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 185d186..1060e99 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -255,9 +255,9 @@ uint64_t bench(
       stderr,
       "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
       params->run_name, bench_name, params->clevel,
-      params->isize, osize / repetitions,
-      repetitions, time_taken, time_taken / repetitions,
-      ((double) 1000 * params->isize * repetitions) / time_taken
+      params->isize, osize / total_repetitions,
+      total_repetitions, time_taken, time_taken / total_repetitions,
+      ((double) 1000 * params->isize * total_repetitions) / time_taken
   );
 
   return time_taken;
-- 
cgit v0.12


From 9d971fd5c3c1e18b225307ef88e529c9b3850fcb Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 14:06:23 -0400
Subject: Switch to Unaligned Samples to Compress Different Blobs Each Time

---
 tests/framebench.c | 53 +++++++++++++++++++++--------------------------------
 1 file changed, 21 insertions(+), 32 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 1060e99..01e30b3 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -28,6 +28,7 @@ typedef struct {
   char *obuf;
   size_t osize;
   const char* ibuf;
+  const char* isample;
   size_t isize;
   size_t num_ibuf;
   char *checkbuf;
@@ -39,13 +40,11 @@ typedef struct {
 } bench_params_t;
 
 size_t compress_frame(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4F_cctx *cctx = p->cctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   const LZ4F_CDict* cdict = p->cdict;
   LZ4F_preferences_t* prefs = p->prefs;
 
@@ -57,7 +56,7 @@ size_t compress_frame(bench_params_t *p) {
     cctx,
     obuf,
     osize,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isample,
     isize,
     cdict,
     prefs);
@@ -67,13 +66,11 @@ size_t compress_frame(bench_params_t *p) {
 }
 
 size_t compress_begin(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4F_cctx *cctx = p->cctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   const LZ4F_CDict* cdict = p->cdict;
   LZ4F_preferences_t* prefs = p->prefs;
   const LZ4F_compressOptions_t* options = p->options;
@@ -90,7 +87,7 @@ size_t compress_begin(bench_params_t *p) {
     cctx,
     obuf,
     oend - obuf,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isample,
     isize,
     options);
   LZ4F_CHECK(oused);
@@ -102,74 +99,61 @@ size_t compress_begin(bench_params_t *p) {
 }
 
 size_t compress_default(bench_params_t *p) {
-  size_t iter = p->iter;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
 
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_default(
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
-      isize, oend - obuf);
+  oused = LZ4_compress_default(isample, obuf, isize, oend - obuf);
   obuf += oused;
 
   return obuf - p->obuf;
 }
 
 size_t compress_extState(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4_stream_t *ctx = p->ctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   int clevel = p->clevel;
 
   char *oend = obuf + osize;
   size_t oused;
 
   oused = LZ4_compress_fast_extState_fastReset(
-      ctx,
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
-      isize, oend - obuf, clevel);
+      ctx, isample, obuf, isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
 }
 
 size_t compress_hc(bench_params_t *p) {
-  size_t iter = p->iter;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   int clevel = p->clevel;
 
   char *oend = obuf + osize;
   size_t oused;
 
   oused = LZ4_compress_HC(
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
-      isize, oend - obuf, clevel);
+      isample, obuf, isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
 }
 
 size_t compress_hc_extState(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4_streamHC_t *hcctx = p->hcctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   int clevel = p->clevel;
 
   char *oend = obuf + osize;
@@ -177,7 +161,7 @@ size_t compress_hc_extState(bench_params_t *p) {
 
   oused = LZ4_compress_HC_extStateHC(
       hcctx,
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isample, obuf,
       isize, oend - obuf, clevel);
   obuf += oused;
 
@@ -189,7 +173,7 @@ size_t check_lz4(bench_params_t *p, size_t csize) {
   memset(p->checkbuf, 0xFF, p->checksize);
   return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize,
                                        p->dictbuf, p->dictsize)
-      && !memcmp(p->ibuf, p->checkbuf, p->isize);
+      && !memcmp(p->isample, p->checkbuf, p->isize);
 }
 
 size_t check_lz4f(bench_params_t *p, size_t csize) {
@@ -211,7 +195,7 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
     dleft = dsize - dp;
     if (LZ4F_isError(ret)) return 0;
   } while (cleft);
-  return !memcmp(p->ibuf, p->checkbuf, p->isize);
+  return !memcmp(p->isample, p->checkbuf, p->isize);
 }
 
 
@@ -236,6 +220,11 @@ uint64_t bench(
 
     for (i = 0; i < repetitions; i++) {
       params->iter = i;
+      if (params->num_ibuf == 1) {
+        params->isample = params->ibuf;
+      } else {
+        params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize));
+      }
       o = fun(params);
       if (!o) return 0;
       osize += o;
@@ -295,7 +284,7 @@ int main(int argc, char *argv[]) {
   LZ4F_preferences_t prefs;
   LZ4F_compressOptions_t options;
 
-  int clevels[] = {1, 2, 3, 6, 9, 10, 12};
+  int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
 
   bench_params_t params;
 
-- 
cgit v0.12


From f646c512e99032cf4545be1ffcfd2a17323fb586 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 18:08:13 -0400
Subject: Print Failure Message in Framebench

---
 tests/framebench.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 01e30b3..048df34 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -226,7 +226,14 @@ uint64_t bench(
         params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize));
       }
       o = fun(params);
-      if (!o) return 0;
+      if (!o) {
+        fprintf(
+            stderr,
+            "%-19s: %-30s @ lvl %2d: %8ld B: FAILED!\n",
+            params->run_name, bench_name, params->clevel,
+            params->isize);
+        return 0;
+      }
       osize += o;
     }
 
@@ -238,7 +245,14 @@ uint64_t bench(
   }
 
   o = checkfun(params, o);
-  if (!o) return 0;
+  if (!o) {
+    fprintf(
+        stderr,
+        "%-19s: %-30s @ lvl %2d: %8ld B: CHECK FAILED!\n",
+        params->run_name, bench_name, params->clevel,
+        params->isize);
+    return 0;
+  }
 
   fprintf(
       stderr,
-- 
cgit v0.12


From 9dae661b1fcc665a78a9de5341e651310d489f5c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 12:39:31 -0400
Subject: Fix Cast

---
 tests/framebench.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 048df34..70cfec5 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -200,7 +200,7 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
 
 
 uint64_t bench(
-    char *bench_name,
+    const char *bench_name,
     size_t (*fun)(bench_params_t *),
     size_t (*checkfun)(bench_params_t *, size_t),
     bench_params_t *params
@@ -299,6 +299,7 @@ int main(int argc, char *argv[]) {
   LZ4F_compressOptions_t options;
 
   int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
+  unsigned int clevelidx;
 
   bench_params_t params;
 
@@ -386,7 +387,7 @@ int main(int argc, char *argv[]) {
   params.prefs = &prefs;
   params.options = &options;
 
-  for (unsigned int clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
+  for (clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
     params.clevel = clevels[clevelidx];
     params.prefs->compressionLevel = clevels[clevelidx];
     params.cdict = NULL;
-- 
cgit v0.12


From e0d8add7917f5eafa4894979b5b68b0db1fea6f4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 18 Apr 2018 15:38:41 -0400
Subject: Fix Framebench Output Buffer Sizing

---
 tests/framebench.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/framebench.c b/tests/framebench.c
index 70cfec5..9752f23 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -347,6 +347,9 @@ int main(int argc, char *argv[]) {
   options.stableSrc = 1;
 
   out_size = LZ4F_compressFrameBound(in_size, &prefs);
+  if ((size_t)LZ4_compressBound(in_size) > out_size) {
+    out_size = LZ4_compressBound(in_size);
+  }
   out_buf = (char *)malloc(out_size);
   if (!out_buf) return 1;
 
-- 
cgit v0.12


From e75153f508073d87e7087e8ef472879773f116f8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 17:47:52 -0400
Subject: Add Debug Log Statements to HC

---
 lib/lz4frame.c | 21 +++++++++++++++++++++
 lib/lz4hc.c    | 18 ++++++++++++++++--
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 507e4fe..0000023 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -95,6 +95,18 @@ You can contact the author at :
 
 #define LZ4F_STATIC_ASSERT(c)    { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
+#  include <stdio.h>
+static int g_debuglog_enable = 1;
+#  define DEBUGLOG(l, ...) {                                  \
+                if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
+                    fprintf(stderr, __FILE__ ": ");           \
+                    fprintf(stderr, __VA_ARGS__);             \
+                    fprintf(stderr, " \n");                   \
+            }   }
+#else
+#  define DEBUGLOG(l, ...)      {}    /* disabled */
+#endif
 
 /*-************************************
 *  Basic Types
@@ -457,6 +469,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
     const char* dictStart = (const char*)dictBuffer;
     LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
+    DEBUGLOG(4, "LZ4F_createCDict(%p) -> %p", dictBuffer, cdict);
     if (!cdict) return NULL;
     if (dictSize > 64 KB) {
         dictStart += dictSize - 64 KB;
@@ -479,6 +492,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 
 void LZ4F_freeCDict(LZ4F_CDict* cdict)
 {
+    DEBUGLOG(4, "LZ4F_freeCDict(%p)", cdict);
     if (cdict==NULL) return;  /* support free on NULL */
     FREEMEM(cdict->dictContent);
     LZ4_freeStream(cdict->fastCtx);
@@ -530,6 +544,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 static void LZ4F_applyCDict(void* ctx,
                             const LZ4F_CDict* cdict,
                             int level) {
+    DEBUGLOG(5, "LZ4F_applyCDict(%p, %p)", ctx, cdict);
     if (level < LZ4HC_CLEVEL_MIN) {
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
@@ -560,6 +575,8 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     BYTE* dstPtr = dstStart;
     BYTE* headerStart;
 
+    DEBUGLOG(4, "LZ4F_compressBegin_usingCDict(%p, %p)", cctxPtr, cdict);
+
     if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
     MEM_INIT(&prefNull, 0, sizeof(prefNull));
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
@@ -778,6 +795,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
+    DEBUGLOG(4, "LZ4F_compressUpdate(%p, %p, %zd)", cctxPtr, srcBuffer, srcSize);
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
@@ -916,6 +934,9 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize,
     BYTE* dstPtr = dstStart;
 
     size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr);
+
+    DEBUGLOG(4, "LZ4F_compressEnd(%p)", cctxPtr);
+
     if (LZ4F_isError(flushSize)) return flushSize;
     dstPtr += flushSize;
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 111a09b..12d26c3 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -86,6 +86,7 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 **************************************/
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
+    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
     hc4->nextToUpdate = 64 KB;
@@ -641,6 +642,8 @@ static int LZ4HC_compress_generic (
         { lz4opt,8192, LZ4_OPT_NUM },  /* 12==LZ4HC_CLEVEL_MAX */
     };
 
+    DEBUGLOG(4, "LZ4HC_compress_generic(%p, %p, %d)", ctx, src, *srcSizePtr);
+
     if (limit == limitedDestSize && dstCapacity < 1) return 0;         /* Impossible to store anything */
     if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0;          /* Unsupported input size (too large or negative) */
 
@@ -706,8 +709,14 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 *  Streaming Functions
 **************************************/
 /* allocation */
-LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
-int             LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+LZ4_streamHC_t* LZ4_createStreamHC(void) {
+    LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
+    DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
+    return LZ4_streamHCPtr;
+}
+
+int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+    DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr);
     if (!LZ4_streamHCPtr) return 0;  /* support free on NULL */
     free(LZ4_streamHCPtr);
     return 0;
@@ -718,6 +727,7 @@ int             LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
 void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
+    DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
@@ -732,6 +742,7 @@ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev
 int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+    DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize);
     if (dictSize > 64 KB) {
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
@@ -747,6 +758,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
 
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
+    DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
     if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
@@ -764,6 +776,7 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
                                             limitedOutput_directive limit)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+    DEBUGLOG(4, "LZ4_compressHC_continue_generic(%p, %p, %d)", LZ4_streamHCPtr, src, *srcSizePtr);
     /* auto-init if forgotten */
     if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src);
 
@@ -812,6 +825,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS
 {
     LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
     int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+    DEBUGLOG(4, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
     if (dictSize > 64 KB) dictSize = 64 KB;
     if (dictSize < 4) dictSize = 0;
     if (dictSize > prefixSize) dictSize = prefixSize;
-- 
cgit v0.12


From f895b9a6c69a059e62de8baca39b878ed1031418 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 13:16:05 -0400
Subject: Add a Dictionary Context Pointer to the HC Context

---
 lib/lz4hc.h | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 7a25bee..1b08425 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -141,7 +141,8 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
 #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #include <stdint.h>
 
-typedef struct
+typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
+struct LZ4HC_CCtx_internal
 {
     uint32_t   hashTable[LZ4HC_HASHTABLESIZE];
     uint16_t   chainTable[LZ4HC_MAXD];
@@ -153,11 +154,13 @@ typedef struct
     uint32_t   lowLimit;        /* below that point, no more dict */
     uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
     int        compressionLevel;
-} LZ4HC_CCtx_internal;
+    const LZ4HC_CCtx_internal* dictCtx;
+};
 
 #else
 
-typedef struct
+typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
+struct LZ4HC_CCtx_internal
 {
     unsigned int   hashTable[LZ4HC_HASHTABLESIZE];
     unsigned short chainTable[LZ4HC_MAXD];
@@ -169,11 +172,12 @@ typedef struct
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
     int            compressionLevel;
-} LZ4HC_CCtx_internal;
+    const LZ4HC_CCtx_internal* dictCtx;
+};
 
 #endif
 
-#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */
+#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 64) /* 262200 */
 #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
 union LZ4_streamHC_u {
     size_t table[LZ4_STREAMHCSIZE_SIZET];
-- 
cgit v0.12


From a992d11fc2204b7cb0a70962aa5911a58ed4918c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 11:21:07 -0400
Subject: Fully Bounds Check Hash Table Reads

---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 12d26c3..6890613 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -222,7 +222,7 @@ LZ4HC_InsertAndGetWiderMatch (
     DEBUGLOG(7, "First match at index %u / %u (lowLimit)",
                 matchIndex, lowLimit);
 
-    while ((matchIndex>=lowLimit) && (nbAttempts)) {
+    while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) {
         DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
         nbAttempts--;
         if (matchIndex >= dictLimit) {
@@ -286,7 +286,7 @@ LZ4HC_InsertAndGetWiderMatch (
                             matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
         }   }   }   }
-    }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
+    }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
     return longest;
 }
-- 
cgit v0.12


From fdeead0b09188213e51e037edcf9c9daf88d25c4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 14:49:47 -0400
Subject: Set dictCtx Rather than memcpy'ing Ctx

---
 lib/lz4frame.c | 9 +++------
 lib/lz4hc.c    | 1 +
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 0000023..821cb49 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -549,12 +549,9 @@ static void LZ4F_applyCDict(void* ctx,
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
-        if (cdict) {
-            memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
-            LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
-        } else {
-            LZ4_resetStreamHC((LZ4_streamHC_t*)(ctx), level);
-        }
+        LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse;
+        LZ4_resetStreamHC((LZ4_streamHC_t*)ctx, level);
+        internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL;
     }
 }
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 6890613..16830c2 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -729,6 +729,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
 
-- 
cgit v0.12


From 6289ff4fb1a560115ffd2c5dff93389d917a6e4e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 16:43:15 -0400
Subject: Call LZ4F_applyCDict Even on NULL CDict

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 821cb49..ea6a668 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -739,8 +739,8 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
 
 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
+    LZ4F_applyCDict(ctx, cdict, level);
     if (cdict) {
-        LZ4F_applyCDict(ctx, cdict, level);
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }
     return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
-- 
cgit v0.12


From 66d217e2400e07fd91753881890722b6dc28ee4b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 16:54:36 -0400
Subject: Perform Lookups into the Dictionary Context

---
 lib/lz4hc.c | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 16830c2..2775713 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -203,6 +203,7 @@ LZ4HC_InsertAndGetWiderMatch (
 {
     U16* const chainTable = hc4->chainTable;
     U32* const HashTable = hc4->hashTable;
+    const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx;
     const BYTE* const base = hc4->base;
     const U32 dictLimit = hc4->dictLimit;
     const BYTE* const lowPrefixPtr = base + dictLimit;
@@ -212,6 +213,7 @@ LZ4HC_InsertAndGetWiderMatch (
     int nbAttempts = maxNbAttempts;
     U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
+    U32 dictMatchIndex;
     repeat_state_e repeat = rep_untested;
     size_t srcPatternLength = 0;
 
@@ -288,6 +290,39 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
+    if (dictCtx != NULL) {
+        ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
+        dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
+        matchIndex = dictMatchIndex + dictIndexDelta;
+        while (dictMatchIndex >= dictCtx->dictLimit && dictMatchIndex + dictCtx->base < dictCtx->end && dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+            const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
+
+            if (LZ4_read32(matchPtr) == pattern) {
+                int mlt;
+                int back = 0;
+                const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
+                if (vLimit > iHighLimit) vLimit = iHighLimit;
+                mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
+                // if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
+                //     mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
+                // }
+                back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
+                mlt -= back;
+                if (mlt > longest) {
+                    longest = mlt;
+                    *matchpos = base + matchIndex + back;
+                    *startpos = ip + back;
+                }
+            }
+
+            {
+                U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex);
+                dictMatchIndex -= nextOffset;
+                matchIndex -= nextOffset;
+            }
+        }
+    }
+
     return longest;
 }
 
-- 
cgit v0.12


From 595ea582890031c30ded483130116ec6a0179a5c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 22 Mar 2018 16:59:50 -0400
Subject: Avoid Resetting Hash Table

---
 lib/lz4hc.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 2775713..b6d9bf0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -87,7 +87,6 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
     hc4->nextToUpdate = 64 KB;
     hc4->base = start - 64 KB;
-- 
cgit v0.12


From b6c35ed6422f60a7a542ed237cadfd9dfe76c219 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 22 Mar 2018 17:00:04 -0400
Subject: Avoid Resetting Chain Table

---
 lib/lz4hc.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b6d9bf0..0caf001 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -87,7 +87,6 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
     hc4->nextToUpdate = 64 KB;
     hc4->base = start - 64 KB;
     hc4->end = start;
-- 
cgit v0.12


From b88a0b4e8812777d1e715ab095738197f7f8eb96 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 26 Mar 2018 11:43:52 -0400
Subject: Only Perform Dict Lookup if Attempts Remain

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 0caf001..e005cd7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -288,7 +288,7 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
-    if (dictCtx != NULL) {
+    if (dictCtx != NULL && nbAttempts) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + dictIndexDelta;
-- 
cgit v0.12


From 4f7b7a8ffa3032a64aa173f7d8bfb7c258316154 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 12:25:05 -0400
Subject: Clear Tables on Dict Load

---
 lib/lz4hc.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index e005cd7..85419f3 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -95,6 +95,12 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
     hc4->lowLimit = 64 KB;
 }
 
+static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
+{
+    MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
+    MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
+}
+
 
 /* Update chains up to ip (excluded) */
 LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
@@ -782,6 +788,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictSize = 64 KB;
     }
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
+    LZ4HC_clearTables (ctxPtr);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
     if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;
-- 
cgit v0.12


From 22db704a73e38a869d846fa52ddcbb0a214a6c43 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 12:26:54 -0400
Subject: Shift Dict Limit Checks out of the Loop

---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 85419f3..8ac650e 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -294,11 +294,11 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
-    if (dictCtx != NULL && nbAttempts) {
+    if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + dictIndexDelta;
-        while (dictMatchIndex >= dictCtx->dictLimit && dictMatchIndex + dictCtx->base < dictCtx->end && dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+        while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {
-- 
cgit v0.12


From 895e76cc20f366ea9f2999abc9ff3e0dd11b4222 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 15:20:47 -0400
Subject: Push Previous Compression Offsets into the Past

---
 lib/lz4hc.c | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 8ac650e..c56d976 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -84,23 +84,29 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 /**************************************
 *  HC Compression
 **************************************/
-static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
-{
-    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    hc4->nextToUpdate = 64 KB;
-    hc4->base = start - 64 KB;
-    hc4->end = start;
-    hc4->dictBase = start - 64 KB;
-    hc4->dictLimit = 64 KB;
-    hc4->lowLimit = 64 KB;
-}
-
 static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 {
+    DEBUGLOG(4, "LZ4HC_clearTables(%p)", hc4);
     MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
 }
 
+static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
+{
+    U32 startingOffset = hc4->end - hc4->base + 64 KB;
+    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
+    if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
+        LZ4HC_clearTables(hc4);
+        startingOffset = 64 KB;
+    }
+    hc4->nextToUpdate = startingOffset;
+    hc4->base = start - startingOffset;
+    hc4->end = start;
+    hc4->dictBase = start - startingOffset;
+    hc4->dictLimit = startingOffset;
+    hc4->lowLimit = startingOffset;
+}
+
 
 /* Update chains up to ip (excluded) */
 LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
@@ -751,6 +757,9 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
     DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
+    LZ4_streamHCPtr->internal_donotuse.end = (void *)-1;
+    LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;
 }
 
@@ -767,6 +776,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
+    LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
-- 
cgit v0.12


From bdd7af6f71c235cea53912d0eec3b94555e9ffde Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 15:59:00 -0400
Subject: Don't Bother Clearing Chain Table for Working Contexts

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c56d976..673b4b3 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -96,7 +96,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
     U32 startingOffset = hc4->end - hc4->base + 64 KB;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
-        LZ4HC_clearTables(hc4);
+        MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
         startingOffset = 64 KB;
     }
     hc4->nextToUpdate = startingOffset;
-- 
cgit v0.12


From a1beba13f7363df4b5d7ca9afe51963176df068d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 16:32:26 -0400
Subject: Reset Stream in LZ4_compress_HC

---
 lib/lz4hc.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 673b4b3..4a52f0c 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -732,7 +732,10 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
 #endif
-    int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
+    int cSize;
+    statePtr->internal_donotuse.end = (void *)-1;
+    LZ4_resetStreamHC(statePtr, compressionLevel);
+    cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
     free(statePtr);
 #endif
-- 
cgit v0.12


From 221211d7d04478d74eec9fd76ccd98a73bd394ee Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 16:32:40 -0400
Subject: Fix Offset Math

---
 lib/lz4hc.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 4a52f0c..5479159 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -93,12 +93,13 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
-    U32 startingOffset = hc4->end - hc4->base + 64 KB;
+    uptrval startingOffset = hc4->end - hc4->base;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
         MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
-        startingOffset = 64 KB;
+        startingOffset = 0;
     }
+    startingOffset += MAX_DISTANCE;
     hc4->nextToUpdate = startingOffset;
     hc4->base = start - startingOffset;
     hc4->end = start;
-- 
cgit v0.12


From 8f9a2db0e1ea492e836dd074dd0ea62720fb6169 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 6 Apr 2018 19:24:22 -0400
Subject: Fix Some Cast/Conversion Warnings

---
 lib/lz4hc.c | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 5479159..11d820f 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -100,12 +100,12 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
         startingOffset = 0;
     }
     startingOffset += MAX_DISTANCE;
-    hc4->nextToUpdate = startingOffset;
+    hc4->nextToUpdate = (U32) startingOffset;
     hc4->base = start - startingOffset;
     hc4->end = start;
     hc4->dictBase = start - startingOffset;
-    hc4->dictLimit = startingOffset;
-    hc4->lowLimit = startingOffset;
+    hc4->dictLimit = (U32) startingOffset;
+    hc4->lowLimit = (U32) startingOffset;
 }
 
 
@@ -304,7 +304,7 @@ LZ4HC_InsertAndGetWiderMatch (
     if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
-        matchIndex = dictMatchIndex + dictIndexDelta;
+        matchIndex = dictMatchIndex + (int)dictIndexDelta;
         while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
@@ -314,9 +314,11 @@ LZ4HC_InsertAndGetWiderMatch (
                 const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
-                // if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
-                //     mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
-                // }
+                /*
+                if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
+                    mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
+                }
+                */
                 back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {
@@ -734,7 +736,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t* const statePtr = &state;
 #endif
     int cSize;
-    statePtr->internal_donotuse.end = (void *)-1;
+    statePtr->internal_donotuse.end = (const BYTE*)-1;
     LZ4_resetStreamHC(statePtr, compressionLevel);
     cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
@@ -761,7 +763,7 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
     DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
-    LZ4_streamHCPtr->internal_donotuse.end = (void *)-1;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;
-- 
cgit v0.12


From 8db291bc1dc95ab58904b6961a5ac0967b148952 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 17:41:15 -0400
Subject: Remove Match Upper Bounds Check

---
 lib/lz4hc.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 11d820f..fa2cbb7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -235,7 +235,7 @@ LZ4HC_InsertAndGetWiderMatch (
     DEBUGLOG(7, "First match at index %u / %u (lowLimit)",
                 matchIndex, lowLimit);
 
-    while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) {
+    while ((matchIndex>=lowLimit) && (nbAttempts)) {
         DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
         nbAttempts--;
         if (matchIndex >= dictLimit) {
@@ -299,7 +299,7 @@ LZ4HC_InsertAndGetWiderMatch (
                             matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
         }   }   }   }
-    }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
+    }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
     if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
@@ -720,6 +720,7 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
+    LZ4_resetStreamHC((LZ4_streamHC_t*)state, compressionLevel);
     LZ4HC_init (ctx, (const BYTE*)src);
     if (dstCapacity < LZ4_compressBound(srcSize))
         return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);
@@ -735,10 +736,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
 #endif
-    int cSize;
-    statePtr->internal_donotuse.end = (const BYTE*)-1;
-    LZ4_resetStreamHC(statePtr, compressionLevel);
-    cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
+    int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
     free(statePtr);
 #endif
@@ -750,6 +748,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
 int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
+    LZ4_resetStreamHC((LZ4_streamHC_t*)LZ4HC_Data, cLevel);
     LZ4HC_init(ctx, (const BYTE*) source);
     return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize);
 }
@@ -782,7 +781,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
-    LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
-- 
cgit v0.12


From 3591fe8ab8b4391d1f9104b38b9ac19deac2a3e8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 16 Apr 2018 15:09:59 -0400
Subject: Add Fast Reset Paths

---
 lib/lz4hc.c | 21 +++++++++++++++++++--
 lib/lz4hc.h |  2 ++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index fa2cbb7..9deb90d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -716,11 +716,11 @@ static int LZ4HC_compress_generic (
 
 int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); }
 
-int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
+int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
-    LZ4_resetStreamHC((LZ4_streamHC_t*)state, compressionLevel);
+    LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel);
     LZ4HC_init (ctx, (const BYTE*)src);
     if (dstCapacity < LZ4_compressBound(srcSize))
         return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);
@@ -728,6 +728,13 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
         return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit);
 }
 
+int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
+{
+    if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
+    LZ4_resetStreamHC ((LZ4_streamHC_t*)state, compressionLevel);
+    return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel);
+}
+
 int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
 {
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
@@ -787,6 +794,16 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
 
+void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
+{
+    LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
+    DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel);
+    LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
+    LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
+}
+
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 1b08425..3b0d0d3 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -270,7 +270,9 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
  */
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
 
 #endif   /* LZ4_HC_SLO_098092834 */
 #endif   /* LZ4_HC_STATIC_LINKING_ONLY */
-- 
cgit v0.12


From 61c7ceffede9607721ca496ac13d788625f9f668 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 16 Apr 2018 15:25:48 -0400
Subject: Use Fast Reset API in LZ4F

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index ea6a668..d087a94 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -743,7 +743,7 @@ static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSi
     if (cdict) {
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }
-    return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
+    return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level);
 }
 
 static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
-- 
cgit v0.12


From 0a2abacd909d0e1e1be9a82de6c2a693264f8afa Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 16 Apr 2018 20:23:19 -0400
Subject: Use Fast Reset in LZ4F Again

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d087a94..a43f595 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -550,7 +550,7 @@ static void LZ4F_applyCDict(void* ctx,
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
         LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse;
-        LZ4_resetStreamHC((LZ4_streamHC_t*)ctx, level);
+        LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
         internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL;
     }
 }
-- 
cgit v0.12


From 22e16d5b509c56ad350ae42664ddcd3da7b97f1f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 18 Apr 2018 13:55:58 -0400
Subject: Split DictCtx-using Code Into Separate Inlining Chain

---
 lib/lz4hc.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 74 insertions(+), 20 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 9deb90d..5c2b001 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -79,6 +79,8 @@
 
 static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
 
+/*===   Enums   ===*/
+typedef enum { noDictCtx, usingDictCtx } dictCtx_directive;
 
 
 /**************************************
@@ -210,7 +212,8 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE** matchpos,
     const BYTE** startpos,
     const int maxNbAttempts,
-    const int patternAnalysis)
+    const int patternAnalysis,
+    const dictCtx_directive dict)
 {
     U16* const chainTable = hc4->chainTable;
     U32* const HashTable = hc4->hashTable;
@@ -301,7 +304,7 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
-    if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
+    if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + (int)dictIndexDelta;
@@ -344,13 +347,14 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
                                  const BYTE* const ip, const BYTE* const iLimit,
                                  const BYTE** matchpos,
                                  const int maxNbAttempts,
-                                 const int patternAnalysis)
+                                 const int patternAnalysis,
+                                 const dictCtx_directive dict)
 {
     const BYTE* uselessPtr = ip;
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis);
+    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict);
 }
 
 
@@ -440,7 +444,8 @@ static int LZ4HC_compress_hashChain (
     int* srcSizePtr,
     int const maxOutputSize,
     unsigned maxNbAttempts,
-    limitedOutput_directive limit
+    const limitedOutput_directive limit,
+    const dictCtx_directive dict
     )
 {
     const int inputSize = *srcSizePtr;
@@ -472,7 +477,7 @@ static int LZ4HC_compress_hashChain (
 
     /* Main Loop */
     while (ip <= mflimit) {
-        ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis);
+        ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict);
         if (ml<MINMATCH) { ip++; continue; }
 
         /* saved, in case we would skip too much */
@@ -484,7 +489,7 @@ _Search2:
         if (ip+ml <= mflimit)
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
-                            maxNbAttempts, patternAnalysis);
+                            maxNbAttempts, patternAnalysis, dict);
         else
             ml2 = ml;
 
@@ -531,7 +536,7 @@ _Search3:
         if (start2 + ml2 <= mflimit)
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
-                            maxNbAttempts, patternAnalysis);
+                            maxNbAttempts, patternAnalysis, dict);
         else
             ml3 = ml2;
 
@@ -655,17 +660,19 @@ static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx,
     const char* const source, char* dst,
     int* srcSizePtr, int dstCapacity,
     int const nbSearches, size_t sufficient_len,
-    limitedOutput_directive limit, int const fullUpdate);
+    const limitedOutput_directive limit, int const fullUpdate,
+    const dictCtx_directive dict);
 
 
-static int LZ4HC_compress_generic (
+LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
     LZ4HC_CCtx_internal* const ctx,
     const char* const src,
     char* const dst,
     int* const srcSizePtr,
     int const dstCapacity,
     int cLevel,
-    limitedOutput_directive limit
+    const limitedOutput_directive limit,
+    const dictCtx_directive dict
     )
 {
     typedef enum { lz4hc, lz4opt } lz4hc_strat_e;
@@ -704,12 +711,57 @@ static int LZ4HC_compress_generic (
         if (cParam.strat == lz4hc)
             return LZ4HC_compress_hashChain(ctx,
                                 src, dst, srcSizePtr, dstCapacity,
-                                cParam.nbSearches, limit);
+                                cParam.nbSearches, limit, dict);
         assert(cParam.strat == lz4opt);
         return LZ4HC_compress_optimal(ctx,
                             src, dst, srcSizePtr, dstCapacity,
                             cParam.nbSearches, cParam.targetLength, limit,
-                            cLevel == LZ4HC_CLEVEL_MAX);  /* ultra mode */
+                            cLevel == LZ4HC_CLEVEL_MAX, dict);  /* ultra mode */
+    }
+}
+
+static int LZ4HC_compress_generic_noDictCtx (
+    LZ4HC_CCtx_internal* const ctx,
+    const char* const src,
+    char* const dst,
+    int* const srcSizePtr,
+    int const dstCapacity,
+    int cLevel,
+    limitedOutput_directive limit
+    )
+{
+    assert(ctx->dictCtx == NULL);
+    return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx);
+}
+
+static int LZ4HC_compress_generic_dictCtx (
+    LZ4HC_CCtx_internal* const ctx,
+    const char* const src,
+    char* const dst,
+    int* const srcSizePtr,
+    int const dstCapacity,
+    int cLevel,
+    limitedOutput_directive limit
+    )
+{
+    assert(ctx->dictCtx != NULL);
+    return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
+}
+
+static int LZ4HC_compress_generic (
+    LZ4HC_CCtx_internal* const ctx,
+    const char* const src,
+    char* const dst,
+    int* const srcSizePtr,
+    int const dstCapacity,
+    int cLevel,
+    limitedOutput_directive limit
+    )
+{
+    if (ctx->dictCtx == NULL) {
+        return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+    } else {
+        return LZ4HC_compress_generic_dictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
     }
 }
 
@@ -1019,7 +1071,8 @@ typedef struct {
 LZ4_FORCE_INLINE LZ4HC_match_t
 LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
                       const BYTE* ip, const BYTE* const iHighLimit,
-                      int minLen, int nbSearches)
+                      int minLen, int nbSearches,
+                      const dictCtx_directive dict)
 {
     LZ4HC_match_t match = { 0 , 0 };
     const BYTE* matchPtr = NULL;
@@ -1028,7 +1081,7 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
     int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
                                 ip, ip, iHighLimit, minLen, &matchPtr, &ip,
-                                nbSearches, 1 /* patternAnalysis */);
+                                nbSearches, 1 /* patternAnalysis */, dict);
     if (matchLength <= minLen) return match;
     match.len = matchLength;
     match.off = (int)(ip-matchPtr);
@@ -1044,8 +1097,9 @@ static int LZ4HC_compress_optimal (
     int dstCapacity,
     int const nbSearches,
     size_t sufficient_len,
-    limitedOutput_directive limit,
-    int const fullUpdate
+    const limitedOutput_directive limit,
+    int const fullUpdate,
+    const dictCtx_directive dict
     )
 {
 #define TRAILING_LITERALS 3
@@ -1073,7 +1127,7 @@ static int LZ4HC_compress_optimal (
          int best_mlen, best_off;
          int cur, last_match_pos = 0;
 
-         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
+         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict);
          if (firstMatch.len==0) { ip++; continue; }
 
          if ((size_t)firstMatch.len > sufficient_len) {
@@ -1143,10 +1197,10 @@ static int LZ4HC_compress_optimal (
 
              DEBUGLOG(7, "search at rPos:%u", cur);
              if (fullUpdate)
-                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict);
              else
                  /* only test matches of minimum length; slightly faster, but misses a few bytes */
-                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict);
              if (!newMatch.len) continue;
 
              if ( ((size_t)newMatch.len > sufficient_len)
-- 
cgit v0.12


From b67de2a327644607c034250d105ba9374b0897e8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 18 Apr 2018 15:52:04 -0400
Subject: Force Inline on HashChain

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 5c2b001..c050c59 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -437,7 +437,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     return 0;
 }
 
-static int LZ4HC_compress_hashChain (
+LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
     LZ4HC_CCtx_internal* const ctx,
     const char* const source,
     char* const dest,
-- 
cgit v0.12


From 0abc23f72e44dd9af86e7fb61a2497cee81ed320 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 13:02:55 -0400
Subject: Copy DictCtx into Working Context on Inputs Larger than 4 KB

---
 lib/lz4hc.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c050c59..c201559 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -720,6 +720,8 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
     }
 }
 
+static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock);
+
 static int LZ4HC_compress_generic_noDictCtx (
     LZ4HC_CCtx_internal* const ctx,
     const char* const src,
@@ -745,7 +747,16 @@ static int LZ4HC_compress_generic_dictCtx (
     )
 {
     assert(ctx->dictCtx != NULL);
-    return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
+    if (*srcSizePtr > 4 KB) {
+        memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
+        LZ4HC_setExternalDict(ctx, (const BYTE *)src);
+        ctx->compressionLevel = cLevel;
+        return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+    } else {
+        int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
+        ctx->dictCtx = NULL;
+        return result;
+    }
 }
 
 static int LZ4HC_compress_generic (
-- 
cgit v0.12


From f4b13e17ea110498c41ba3ac8dc6df0438c37d29 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 17:51:10 -0400
Subject: Don't Clear the Dictionary Context Until No Longer Useful

---
 lib/lz4hc.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c201559..9a02787 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -746,15 +746,18 @@ static int LZ4HC_compress_generic_dictCtx (
     limitedOutput_directive limit
     )
 {
+    size_t position = ctx->end - ctx->base - ctx->lowLimit;
     assert(ctx->dictCtx != NULL);
-    if (*srcSizePtr > 4 KB) {
+    if (position >= 64 KB) {
+        ctx->dictCtx = NULL;
+        return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+    } else if (position == 0 && *srcSizePtr > 4 KB) {
         memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
         LZ4HC_setExternalDict(ctx, (const BYTE *)src);
         ctx->compressionLevel = cLevel;
         return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
     } else {
         int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
-        ctx->dictCtx = NULL;
         return result;
     }
 }
-- 
cgit v0.12


From 14c577d4c9332197dc0003833e043608b4b4b239 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 19:36:34 -0400
Subject: Fix Signedness of Comparison

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 9a02787..20df8fa 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -308,7 +308,7 @@ LZ4HC_InsertAndGetWiderMatch (
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + (int)dictIndexDelta;
-        while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+        while ((ptrdiff_t) dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {
-- 
cgit v0.12


From 0064e8ebc7a475d27ba658ca92e40de75c9cac72 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 13:14:37 -0400
Subject: Remove Commented Out Support for Match Continuation over Segment
 Boundary

---
 lib/lz4hc.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 20df8fa..e09d8e0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -317,11 +317,6 @@ LZ4HC_InsertAndGetWiderMatch (
                 const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
-                /*
-                if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
-                    mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
-                }
-                */
                 back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {
-- 
cgit v0.12


From 8f118cf6e9f3030f52414177d93ab447e8e24128 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 14:08:06 -0400
Subject: Remove inputBuffer from Context, Work Around its Absence

---
 lib/lz4hc.c | 13 +++++++------
 lib/lz4hc.h | 11 ++++++++---
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index e09d8e0..dff42f0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -999,8 +999,8 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
 {
     LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1;   /* Error : pointer is not aligned for pointer (32 or 64 bits) */
+    LZ4_resetStreamHC((LZ4_streamHC_t*)state, ((LZ4_streamHC_t*)state)->internal_donotuse.compressionLevel);
     LZ4HC_init(ctx, (const BYTE*)inputBuffer);
-    ctx->inputBuffer = inputBuffer;
     return 0;
 }
 
@@ -1008,9 +1008,8 @@ void* LZ4_createHC (const char* inputBuffer)
 {
     LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
     if (hc4 == NULL) return NULL;   /* not enough memory */
+    LZ4_resetStreamHC(hc4, 0 /* compressionLevel */);
     LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
-    assert(sizeof(size_t) == sizeof(void*));
-    hc4->internal_donotuse.inputBuffer = (void*)(size_t)inputBuffer;   /* ugly hack, circumvent -Wcast-qual */
     return hc4;
 }
 
@@ -1032,9 +1031,11 @@ int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, c
 
 char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
 {
-    LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
-    int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
-    return (char*)(hc4->inputBuffer) + dictSize;
+    LZ4_streamHC_t *ctx = (LZ4_streamHC_t*)LZ4HC_Data;
+    const BYTE *bufferStart = ctx->internal_donotuse.base + ctx->internal_donotuse.lowLimit;
+    LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel);
+    /* avoid const char * -> char * conversion warning :( */
+    return (char *)(uptrval)bufferStart;
 }
 
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 3b0d0d3..cfc5d9e 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -149,7 +149,6 @@ struct LZ4HC_CCtx_internal
     const uint8_t* end;         /* next block here to continue on current prefix */
     const uint8_t* base;        /* All index relative to this position */
     const uint8_t* dictBase;    /* alternate base for extDict */
-    void*      inputBuffer;     /* deprecated */
     uint32_t   dictLimit;       /* below that point, need extDict */
     uint32_t   lowLimit;        /* below that point, no more dict */
     uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
@@ -167,7 +166,6 @@ struct LZ4HC_CCtx_internal
     const unsigned char* end;        /* next block here to continue on current prefix */
     const unsigned char* base;       /* All index relative to this position */
     const unsigned char* dictBase;   /* alternate base for extDict */
-    void*          inputBuffer;      /* deprecated */
     unsigned int   dictLimit;        /* below that point, need extDict */
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
@@ -210,7 +208,14 @@ LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
-/* Deprecated Streaming functions; should no longer be used */
+/* Obsolete streaming functions; degraded functionality; do not use!
+ *
+ * In order to perform streaming compression, these functions depended on data
+ * that is no longer tracked in the state. They have been preserved as well as
+ * possible: using them will still produce a correct output. However, use of
+ * LZ4_slideInputBufferHC() will truncate the history of the stream, rather
+ * than preserve a window-sized chunk of history.
+ */
 LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);
 LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
 LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API   int   LZ4_freeHC (void* LZ4HC_Data);
-- 
cgit v0.12


From ca833f928f64d0b88fbfd2a8a8bc11b8333e487d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 14:16:27 -0400
Subject: Also Reset the Chain Table

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index dff42f0..124da86 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -98,7 +98,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
     uptrval startingOffset = hc4->end - hc4->base;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
-        MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
+        LZ4HC_clearTables(hc4);
         startingOffset = 0;
     }
     startingOffset += MAX_DISTANCE;
-- 
cgit v0.12


From d7347f9eeaffe5351886b6174f9974bd541b9b7d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 14:52:13 -0400
Subject: Add API for Attaching Dictionaries

---
 lib/lz4frame.c |  3 +--
 lib/lz4hc.c    |  3 +++
 lib/lz4hc.h    | 27 +++++++++++++++++++++++++++
 3 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a43f595..d4d9397 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -549,9 +549,8 @@ static void LZ4F_applyCDict(void* ctx,
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
-        LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse;
         LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
-        internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL;
+        LZ4_attach_HC_dictionary((LZ4_streamHC_t *)ctx, cdict ? cdict->HCCtx : NULL);
     }
 }
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 124da86..690015f 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -887,6 +887,9 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
     return dictSize;
 }
 
+void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream) {
+    working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL;
+}
 
 /* compression */
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index cfc5d9e..fb8397e 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -279,5 +279,32 @@ void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLeve
 
 int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
 
+/*! LZ4_attach_HC_dictionary() :
+ *  This is an experimental API that allows for the efficient use of a
+ *  static dictionary many times.
+ *
+ *  Rather than re-loading the dictionary buffer into a working context before
+ *  each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a
+ *  working LZ4_streamHC_t, this function introduces a no-copy setup mechanism,
+ *  in which the working stream references the dictionary stream in-place.
+ *
+ *  Several assumptions are made about the state of the dictionary stream.
+ *  Currently, only streams which have been prepared by LZ4_loadDictHC() should
+ *  be expected to work.
+ *
+ *  Alternatively, the provided dictionary stream pointer may be NULL, in which
+ *  case any existing dictionary stream is unset.
+ *
+ *  A dictionary should only be attached to a stream without any history (i.e.,
+ *  a stream that has just been reset).
+ *
+ *  The dictionary will remain attached to the working stream only for the
+ *  current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the
+ *  dictionary context association from the working stream. The dictionary
+ *  stream (and source buffer) must remain in-place / accessible / unchanged
+ *  through the lifetime of the stream session.
+ */
+LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream);
+
 #endif   /* LZ4_HC_SLO_098092834 */
 #endif   /* LZ4_HC_STATIC_LINKING_ONLY */
-- 
cgit v0.12


From 3f087cf1cbcb0e0082ead149a621d259cb57e7ba Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 15:00:53 -0400
Subject: Add Comments on New Public APIs

---
 lib/lz4hc.h | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index fb8397e..a762d3f 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -275,8 +275,37 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
  */
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+/*! LZ4_resetStreamHC_fast() :
+ *  When an LZ4_streamHC_t is known to be in a internally coherent state,
+ *  it can often be prepared for a new compression with almost no work, only
+ *  sometimes falling back to the full, expensive reset that is always required
+ *  when the stream is in an indeterminate state (i.e., the reset performed by
+ *  LZ4_resetStreamHC()).
+ *
+ *  LZ4_streamHCs are guaranteed to be in a valid state when:
+ *  - returned from LZ4_createStreamHC()
+ *  - reset by LZ4_resetStreamHC()
+ *  - memset(stream, 0, sizeof(LZ4_streamHC_t))
+ *  - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast()
+ *  - the stream was in a valid state and was then used in any compression call
+ *    that returned success
+ *  - the stream was in an indeterminate state and was used in a compression
+ *    call that fully reset the state (LZ4_compress_HC_extStateHC()) and that
+ *    returned success
+ */
 void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+/*! LZ4_compress_HC_extStateHC_fastReset() :
+ *  A variant of LZ4_compress_HC_extStateHC().
+ *
+ *  Using this variant avoids an expensive initialization step. It is only safe
+ *  to call if the state buffer is known to be correctly initialized already
+ *  (see above comment on LZ4_resetStreamHC_fast() for a definition of
+ *  "correctly initialized"). From a high level, the difference is that this
+ *  function initializes the provided state with a call to
+ *  LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a
+ *  call to LZ4_resetStreamHC().
+ */
 int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
 
 /*! LZ4_attach_HC_dictionary() :
-- 
cgit v0.12


From 209c9c29d1baf5b2f4fb1fc38b2612dd95d1b9a1 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 15:16:41 -0400
Subject: Add Some Simple Fuzzer Tests

---
 tests/fuzzer.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 244cc4f..8b21f8e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -446,7 +446,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /* Test compression HC using external state */
         FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()");
         ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed");
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed")
+
+        /* Test compression HC using fast reset external state */
+        FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC_fastReset()");
+        ret = LZ4_compress_HC_extStateHC_fastReset(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC_fastReset() failed");
 
         /* Test compression using external state */
         FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()");
@@ -810,6 +815,49 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
         }
 
+        /* Compress HC using external dictionary stream */
+        FUZ_DISPLAYTEST();
+        {
+            LZ4_streamHC_t LZ4_streamHC;
+
+            LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
+            LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+            LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
+            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed");
+
+            FUZ_DISPLAYTEST();
+            LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+            FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
+
+            FUZ_DISPLAYTEST();
+            LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer");
+
+            FUZ_DISPLAYTEST();
+            LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer");
+
+            FUZ_DISPLAYTEST();
+            decodedBuffer[blockSize] = 0;
+            ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
+            FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
+            FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+            {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+                if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+                FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+            }
+        }
+
         /* Compress HC continue destSize */
         FUZ_DISPLAYTEST();
         {   int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5;
-- 
cgit v0.12


From 7874cf06b3307a7ba5efdd141d068887480aaf11 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 15:30:08 -0400
Subject: Consts and Asserts and Other Minor Nits

---
 lib/lz4hc.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 690015f..e5eb11d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -305,10 +305,13 @@ LZ4HC_InsertAndGetWiderMatch (
     }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
     if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
-        ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
+        const ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
+        /* bounds check, since we need to downcast */
+        assert(dictIndexDelta <= 1 GB);
+        assert(dictIndexDelta >= -1 GB);
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + (int)dictIndexDelta;
-        while ((ptrdiff_t) dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+        while ((ptrdiff_t) matchIndex + MAX_DISTANCE > ip - base && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {
@@ -741,7 +744,7 @@ static int LZ4HC_compress_generic_dictCtx (
     limitedOutput_directive limit
     )
 {
-    size_t position = ctx->end - ctx->base - ctx->lowLimit;
+    const size_t position = ctx->end - ctx->base - ctx->lowLimit;
     assert(ctx->dictCtx != NULL);
     if (position >= 64 KB) {
         ctx->dictCtx = NULL;
@@ -752,8 +755,7 @@ static int LZ4HC_compress_generic_dictCtx (
         ctx->compressionLevel = cLevel;
         return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
     } else {
-        int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
-        return result;
+        return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
     }
 }
 
@@ -804,7 +806,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
 #endif
-    int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
+    int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
     free(statePtr);
 #endif
-- 
cgit v0.12


From 1d2500d44e97a46eb447745d02652e02435baadb Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 17:10:47 -0400
Subject: Handle Index Underflows Safely

---
 lib/lz4hc.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index e5eb11d..843b539 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -221,7 +221,8 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE* const base = hc4->base;
     const U32 dictLimit = hc4->dictLimit;
     const BYTE* const lowPrefixPtr = base + dictLimit;
-    const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - MAX_DISTANCE;
+    const U32 ipIndex = (U32)(ip - base);
+    const U32 lowLimit = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
     const BYTE* const dictBase = hc4->dictBase;
     int const delta = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
@@ -304,14 +305,12 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
-    if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
-        const ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
-        /* bounds check, since we need to downcast */
-        assert(dictIndexDelta <= 1 GB);
-        assert(dictIndexDelta >= -1 GB);
+    if (dict == usingDictCtx && nbAttempts && ipIndex - lowLimit < MAX_DISTANCE) {
+        size_t const dictEndOffset = dictCtx->end - dictCtx->base;
+        assert(dictEndOffset <= 1 GB);
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
-        matchIndex = dictMatchIndex + (int)dictIndexDelta;
-        while ((ptrdiff_t) matchIndex + MAX_DISTANCE > ip - base && nbAttempts--) {
+        matchIndex = dictMatchIndex + lowLimit - (U32)dictEndOffset;
+        while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {
-- 
cgit v0.12


From 86b381e40b6ec3394582a2113c7bc80f1d619bab Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 17:13:40 -0400
Subject: Fix Constant Value

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 843b539..48c42c6 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -101,7 +101,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
         LZ4HC_clearTables(hc4);
         startingOffset = 0;
     }
-    startingOffset += MAX_DISTANCE;
+    startingOffset += 64 KB;
     hc4->nextToUpdate = (U32) startingOffset;
     hc4->base = start - startingOffset;
     hc4->end = start;
-- 
cgit v0.12


From 756ed402da8161c7bc3749bebaac069259312d8b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 17:56:26 -0400
Subject: Sign-Extend -1 to Pointer Width

---
 lib/lz4hc.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 48c42c6..a973086 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -830,8 +830,7 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;
@@ -850,7 +849,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
-- 
cgit v0.12


From 85cac61dd8823886132b4a9c2ff4a43b237da917 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 19:35:51 -0400
Subject: Don't Segfault on Malloc Failure

---
 lib/lz4hc.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index a973086..b0325a7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -830,9 +830,11 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
-    LZ4_streamHCPtr->internal_donotuse.base = NULL;
-    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    if (LZ4_streamHCPtr != NULL) {
+        LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
+        LZ4_streamHCPtr->internal_donotuse.base = NULL;
+        LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    }
     return LZ4_streamHCPtr;
 }
 
-- 
cgit v0.12


From a8cb2feffdead7b49a09aad0dc7b13dcff206e5b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 19:37:07 -0400
Subject: Tolerate Base Pointer Underflow

---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b0325a7..b70d03b 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -97,7 +97,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     uptrval startingOffset = hc4->end - hc4->base;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
+    if (startingOffset > 1 GB) {
         LZ4HC_clearTables(hc4);
         startingOffset = 0;
     }
@@ -898,7 +898,7 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
     DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
-    if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
+    if (ctxPtr->end - ctxPtr->base - ctxPtr->nextToUpdate >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
     ctxPtr->lowLimit  = ctxPtr->dictLimit;
-- 
cgit v0.12


From fcc99d1f31bf8a913fa91a5455524b4a14b9a03d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 19:37:28 -0400
Subject: Simpler loadDict() Reset

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b70d03b..23d84f4 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -882,8 +882,8 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
     }
+    LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4_streamHCPtr->internal_donotuse.compressionLevel);
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
-    LZ4HC_clearTables (ctxPtr);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
     if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;
-- 
cgit v0.12


From 1895fa19a4f04b395f9ebb1fe38049ab4c909fc3 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 20:14:12 -0400
Subject: Remove Redundant Static Assert

---
 lib/lz4hc.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 23d84f4..f91ef4a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -859,7 +859,6 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 
 void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
-    LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel);
     LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
-- 
cgit v0.12


From ee67f25576f111972bd5003c527d1ebee598dc71 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 20:18:30 -0400
Subject: Change vLimit Calculation

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f91ef4a..a12f298 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -316,7 +316,7 @@ LZ4HC_InsertAndGetWiderMatch (
             if (LZ4_read32(matchPtr) == pattern) {
                 int mlt;
                 int back = 0;
-                const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
+                const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
                 back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
-- 
cgit v0.12


From a8a5dfd426cec6e3ac6c689a314aff5817de9ce3 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 20 Apr 2018 18:09:51 -0700
Subject: fixed clang performance in lz4_fast

The simple change from
`matchIndex+MAX_DISTANCE < current`
towards
`current - matchIndex > MAX_DISTANCE`

is enough to generate a 10% performance drop under clang.
Quite massive.
(I missed as my eyes were concentrated on gcc performance at that time).

The second version is more robust, because it also survives a situation where
`matchIndex > current`
due to overflows.

The first version requires matchIndex to not overflow.
Hence were added `assert()` conditions.

The only case where this can happen is with dictCtx compression,
in the case where the dictionary context is not initialized before loading the dictionary.
So it's enough to always initialize the context while loading the dictionary.
---
 lib/lz4.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index bb6b619..3ca1126 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -777,7 +777,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
 
                 if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;     /* match outside of valid area */
-                if ((tableType != byU16) && (current - matchIndex > MAX_DISTANCE)) continue; /* too far - note: works even if matchIndex overflows */
+                assert(matchIndex < current);
+                if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far - note: works even if matchIndex overflows */
                 if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);      /* too_far presumed impossible with byU16 */
 
                 if (LZ4_read32(match) == LZ4_read32(ip)) {
@@ -918,8 +919,9 @@ _next_match:
                 match = base + matchIndex;
             }
             LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
+            assert(matchIndex < current);
             if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
-              && ((tableType==byU16) ? 1 : (current - matchIndex <= MAX_DISTANCE))
+              && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current))
               && (LZ4_read32(match) == LZ4_read32(ip)) ) {
                 token=op++;
                 *token=0;
@@ -1304,7 +1306,11 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 
     DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict);
 
-    LZ4_prepareTable(dict, 0, tableType);
+    /* It's necessary to reset the context,
+     * and not just continue it with prepareTable()
+     * to avoid any risk of generating overflowing matchIndex
+     * when compressing using this dictionary */
+    LZ4_resetStream(LZ4_dict);
 
     /* We always increment the offset by 64 KB, since, if the dict is longer,
      * we truncate it to the last 64k, and if it's shorter, we still want to
-- 
cgit v0.12


From d1f21883d652ac0abae91bc04924747c82bdbabd Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Sat, 21 Apr 2018 00:11:51 -0700
Subject: fixed incorrect comment

---
 lib/lz4.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 3ca1126..2205162 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -776,10 +776,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
 
-                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;     /* match outside of valid area */
+                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;    /* match outside of valid area */
                 assert(matchIndex < current);
-                if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far - note: works even if matchIndex overflows */
-                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);      /* too_far presumed impossible with byU16 */
+                if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue;  /* too far */
+                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);     /* too_far presumed impossible with byU16 */
 
                 if (LZ4_read32(match) == LZ4_read32(ip)) {
                     if (maybe_extMem) offset = current - matchIndex;
-- 
cgit v0.12


From ab06ef97bb5bb50f642add1e80854a09c3d38068 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Mon, 23 Apr 2018 01:43:30 +0300
Subject: lz4.h: clarify the risks of using LZ4_decompress_fast()

The notes about "security guarantee" and "malicious inputs" seemed
a bit non-technical to me, so I took the liberty to tone them down
and instead describe the actual risks in technical terms.  Namely,
the function never writes past the end of the output buffer, so
a direct hostile takeover (resulting in arbitrary code execution
soon after the return from the function) is not possible.  However,
the application can crash because of reads from unmapped pages.

I also took the liberty to describe what I believe is the only sensible
usage scenario for the function: "This function is only usable if the
originalSize of uncompressed data is known in advance," etc.
---
 lib/lz4.h | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 0dfa19e..db6353c 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -206,15 +206,17 @@ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePt
 /*!
 LZ4_decompress_fast() : **unsafe!**
 This function is a bit faster than LZ4_decompress_safe(),
-but doesn't provide any security guarantee.
+but it may misbehave on malformed input because it doesn't perform full validation of compressed data.
     originalSize : is the uncompressed size to regenerate
                    Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
     return : number of bytes read from source buffer (== compressed size).
              If the source stream is detected malformed, the function stops decoding and return a negative result.
-    note : This function respects memory boundaries for *properly formed* compressed data.
-           However, it does not provide any protection against malicious input.
-           It also doesn't know 'src' size, and implies it's >= compressed size.
-           Use this function in trusted environment **only**.
+    note : This function is only usable if the originalSize of uncompressed data is known in advance.
+           The caller should also check that all the compressed input has been consumed properly,
+           i.e. that the return value matches the size of the buffer with compressed input.
+           The function never writes past the output buffer.  However, since it doesn't know its 'src' size,
+           it may read past the intended input.  Also, because match offsets are not validated during decoding,
+           reads from 'src' may underflow.  Use this function in trusted environment **only**.
 */
 LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
 
-- 
cgit v0.12


From bb83cad98fdb15a7ade4cde582b98e836fb8ef11 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 23 Apr 2018 13:14:19 -0700
Subject: Fix input size validation edge cases

The bug is a read up to 2 bytes past the end of the buffer.
There are three cases for this bug, one for each test case added.

* An empty input causes `token = *ip++` to read one byte too far.
* A one byte input with `(token >> ML_BITS) == RUN_MASK` causes
  one extra byte to be read without validation. This could be
  combined with the first bug to cause 2 extra bytes to be read.
* The case pointed out in issue #508, where `ip == iend` at the
  beginning of the loop after taking the shortcut.

Benchmarks show no regressions on clang or gcc-7 on both my mac
and devserver.

Fixes #508.
---
 lib/lz4.c      |  8 ++++++--
 tests/fuzzer.c | 26 ++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index bb6b619..870ab5a 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1520,6 +1520,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
     if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT;                      /* targetOutputSize too high => just decode everything */
     if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1;  /* Empty output buffer */
     if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
+    if ((endOnInput) && unlikely(srcSize==0)) return -1;
 
     /* Main Loop : decode sequences */
     while (1) {
@@ -1529,11 +1530,13 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 
         unsigned const token = *ip++;
 
+        assert(ip <= iend); /* ip < iend before the increment */
         /* shortcut for common case :
          * in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes).
          * this shortcut was tested on x86 and x64, where it improves decoding speed.
-         * it has not yet been benchmarked on ARM, Power, mips, etc. */
-        if (((ip + 14 /*maxLL*/ + 2 /*offset*/ <= iend)
+         * it has not yet been benchmarked on ARM, Power, mips, etc.
+         * NOTE: The loop begins with a read, so we must have one byte left at the end. */
+        if (((ip + 14 /*maxLL*/ + 2 /*offset*/ < iend)
           & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend))
           & ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) {
             size_t const ll = token >> ML_BITS;
@@ -1553,6 +1556,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
         /* decode literal length */
         if ((length=(token>>ML_BITS)) == RUN_MASK) {
             unsigned s;
+            if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error;   /* overflow detection */
             do {
                 s = *ip++;
                 length += s;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 244cc4f..def5230 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -487,6 +487,32 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
 
+        /* Test decoding with empty input */
+        FUZ_DISPLAYTEST("LZ4_decompress_safe() with empty input");
+        LZ4_decompress_safe(NULL, decodedBuffer, 0, blockSize);
+
+        /* Test decoding with a one byte input */
+        FUZ_DISPLAYTEST("LZ4_decompress_safe() with one byte input");
+        {   char const tmp = 0xFF;
+            LZ4_decompress_safe(&tmp, decodedBuffer, 1, blockSize);
+        }
+
+        /* Test decoding shortcut edge case */
+        FUZ_DISPLAYTEST("LZ4_decompress_safe() with shortcut edge case");
+        {   char tmp[17];
+            unsigned long i;
+            /* 14 bytes of literals, followed by a 14 byte match.
+             * Should not read beyond the end of the buffer.
+             * See https://github.com/lz4/lz4/issues/508. */
+            *tmp = 0xEE;
+            memset(tmp + 1, 0, 14);
+            tmp[15] = 14;
+            tmp[16] = 0;
+            ret = LZ4_decompress_safe(tmp, decodedBuffer, sizeof(tmp), blockSize);
+            FUZ_CHECKTEST(ret >= 0, "LZ4_decompress_safe() should fail");
+        }
+
+
         /* Test decoding with output size exactly what's necessary => must work */
         FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
-- 
cgit v0.12


From 672799e8149c4c9b89f767801209721a59012c91 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 23 Apr 2018 14:21:02 -0700
Subject: Fix compilation error and assert.

---
 lib/lz4.c      | 2 +-
 tests/fuzzer.c | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 870ab5a..40b2229 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1530,7 +1530,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 
         unsigned const token = *ip++;
 
-        assert(ip <= iend); /* ip < iend before the increment */
+        assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
         /* shortcut for common case :
          * in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes).
          * this shortcut was tested on x86 and x64, where it improves decoding speed.
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index def5230..6fd27fc 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -500,7 +500,6 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /* Test decoding shortcut edge case */
         FUZ_DISPLAYTEST("LZ4_decompress_safe() with shortcut edge case");
         {   char tmp[17];
-            unsigned long i;
             /* 14 bytes of literals, followed by a 14 byte match.
              * Should not read beyond the end of the buffer.
              * See https://github.com/lz4/lz4/issues/508. */
-- 
cgit v0.12


From bd06fde104c7c29b8d748dad8f27a08c328b74fe Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 23 Apr 2018 15:42:27 -0700
Subject: fullbench compiled without assert()

to better reflect release speed
---
 lib/Makefile   | 1 +
 tests/Makefile | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/Makefile b/lib/Makefile
index 7b31239..673406d 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -181,6 +181,7 @@ uninstall:
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
+	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
 	@echo lz4 libraries successfully uninstalled
 
 endif
diff --git a/tests/Makefile b/tests/Makefile
index 2b93c9f..d4847b1 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -80,7 +80,8 @@ lz4c32:   # create a 32-bits version for 32/64 interop tests
 %.o : $(LZ4DIR)/%.c $(LZ4DIR)/%.h
 	$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
 
-fullbench  : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c
+fullbench : DEBUGLEVEL=0
+fullbench : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 $(LZ4DIR)/liblz4.a:
-- 
cgit v0.12


From cd0663456f82b6d771e6aed01c6c4ff7f1bf4358 Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 23 Apr 2018 15:47:08 -0700
Subject: disable shortcut for LZ4_decompress_fast()

improving speed
---
 lib/lz4.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 40b2229..d794c35 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1536,9 +1536,10 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
          * this shortcut was tested on x86 and x64, where it improves decoding speed.
          * it has not yet been benchmarked on ARM, Power, mips, etc.
          * NOTE: The loop begins with a read, so we must have one byte left at the end. */
-        if (((ip + 14 /*maxLL*/ + 2 /*offset*/ < iend)
-          & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend))
-          & ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) {
+        if (endOnInput
+          && ((ip + 14 /*maxLL*/ + 2 /*offset*/ < iend)
+            & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend))
+            & ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) {
             size_t const ll = token >> ML_BITS;
             size_t const off = LZ4_readLE16(ip+ll);
             const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */
-- 
cgit v0.12


From 644b7bd2b62b10da2b464d71655ae2ed523fdc33 Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 23 Apr 2018 15:52:44 -0700
Subject: fixed minor declaration issue with clang on msys

---
 programs/util.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/programs/util.h b/programs/util.h
index ff25106..ef6ca77 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -30,8 +30,9 @@ extern "C" {
 *  Dependencies
 ******************************************/
 #include "platform.h"     /* PLATFORM_POSIX_VERSION */
-#include <stdlib.h>       /* malloc */
 #include <stddef.h>       /* size_t, ptrdiff_t */
+#include <stdlib.h>       /* malloc */
+#include <string.h>       /* strlen, strncpy */
 #include <stdio.h>        /* fprintf */
 #include <sys/types.h>    /* stat, utime */
 #include <sys/stat.h>     /* stat */
-- 
cgit v0.12


From 44bff3fd3b4fc6280cad3f8b930d6a404e3bb236 Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 23 Apr 2018 19:26:02 -0700
Subject: re-ordered parenthesis

to avoid mixing && and &
as suggested by @terrelln
---
 lib/lz4.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d794c35..b2e08e3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1538,8 +1538,9 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
          * NOTE: The loop begins with a read, so we must have one byte left at the end. */
         if (endOnInput
           && ((ip + 14 /*maxLL*/ + 2 /*offset*/ < iend)
-            & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend))
-            & ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) {
+            & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend)
+            & (token < (15<<ML_BITS))
+            & ((token & ML_MASK) != 15) ) ) {
             size_t const ll = token >> ML_BITS;
             size_t const off = LZ4_readLE16(ip+ll);
             const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */
-- 
cgit v0.12


From 13271a88d7c21bee4ee61f619ef359afc707f23c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 24 Apr 2018 11:55:53 -0400
Subject: Revert Stream Size Const to Correct Value

---
 lib/lz4hc.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index a762d3f..28e2528 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -175,7 +175,7 @@ struct LZ4HC_CCtx_internal
 
 #endif
 
-#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 64) /* 262200 */
+#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */
 #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
 union LZ4_streamHC_u {
     size_t table[LZ4_STREAMHCSIZE_SIZET];
-- 
cgit v0.12


From db9deb7b747d630a5778ed588d88dae82328dc62 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 18:56:01 -0400
Subject: Remove the Framebench Tool

---
 tests/Makefile     |   6 +-
 tests/framebench.c | 412 -----------------------------------------------------
 2 files changed, 1 insertion(+), 417 deletions(-)
 delete mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index f6a4ff3..2b93c9f 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -63,7 +63,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen framebench
+all: fullbench fuzzer frametest datagen
 
 all32: CFLAGS+=-m32
 all32: all
@@ -99,9 +99,6 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
-	$(CC) $(FLAGS) $^ -o $@$(EXT)
-
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -113,7 +110,6 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
-        framebench$(EXT) \
         fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
diff --git a/tests/framebench.c b/tests/framebench.c
deleted file mode 100644
index 9752f23..0000000
--- a/tests/framebench.c
+++ /dev/null
@@ -1,412 +0,0 @@
-#include <assert.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <time.h>
-#include <unistd.h>
-
-#define LZ4_STATIC_LINKING_ONLY
-#include "lz4.h"
-#include "lz4hc.h"
-#include "lz4frame.h"
-#include "lz4frame_static.h"
-
-#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
-
-typedef struct {
-  const char *run_name;
-  size_t iter;
-  LZ4_stream_t *ctx;
-  LZ4_streamHC_t *hcctx;
-  LZ4F_cctx *cctx;
-  LZ4F_dctx *dctx;
-  const char *dictbuf;
-  size_t dictsize;
-  char *obuf;
-  size_t osize;
-  const char* ibuf;
-  const char* isample;
-  size_t isize;
-  size_t num_ibuf;
-  char *checkbuf;
-  size_t checksize;
-  int clevel;
-  const LZ4F_CDict* cdict;
-  LZ4F_preferences_t* prefs;
-  const LZ4F_compressOptions_t* options;
-} bench_params_t;
-
-size_t compress_frame(bench_params_t *p) {
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressFrame_usingCDict(
-    cctx,
-    obuf,
-    osize,
-    isample,
-    isize,
-    cdict,
-    prefs);
-  LZ4F_CHECK(oused);
-
-  return oused;
-}
-
-size_t compress_begin(bench_params_t *p) {
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-  const LZ4F_compressOptions_t* options = p->options;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressUpdate(
-    cctx,
-    obuf,
-    oend - obuf,
-    isample,
-    isize,
-    options);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
-  LZ4F_CHECK(oused);
-
-  return obuf - p->obuf;
-}
-
-size_t compress_default(bench_params_t *p) {
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_default(isample, obuf, isize, oend - obuf);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_extState(bench_params_t *p) {
-  LZ4_stream_t *ctx = p->ctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  int clevel = p->clevel;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_fast_extState_fastReset(
-      ctx, isample, obuf, isize, oend - obuf, clevel);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_hc(bench_params_t *p) {
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  int clevel = p->clevel;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_HC(
-      isample, obuf, isize, oend - obuf, clevel);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_hc_extState(bench_params_t *p) {
-  LZ4_streamHC_t *hcctx = p->hcctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  int clevel = p->clevel;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_HC_extStateHC(
-      hcctx,
-      isample, obuf,
-      isize, oend - obuf, clevel);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t check_lz4(bench_params_t *p, size_t csize) {
-  (void)csize;
-  memset(p->checkbuf, 0xFF, p->checksize);
-  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize,
-                                       p->dictbuf, p->dictsize)
-      && !memcmp(p->isample, p->checkbuf, p->isize);
-}
-
-size_t check_lz4f(bench_params_t *p, size_t csize) {
-  size_t cp = 0;
-  size_t dp = 0;
-  size_t dsize = p->checksize;
-  size_t cleft = csize;
-  size_t dleft = dsize;
-  size_t ret;
-  memset(p->checkbuf, 0xFF, p->checksize);
-  LZ4F_resetDecompressionContext(p->dctx);
-  do {
-    ret = LZ4F_decompress_usingDict(
-        p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft,
-        p->dictbuf, p->dictsize, NULL);
-    cp += cleft;
-    dp += dleft;
-    cleft = csize - cp;
-    dleft = dsize - dp;
-    if (LZ4F_isError(ret)) return 0;
-  } while (cleft);
-  return !memcmp(p->isample, p->checkbuf, p->isize);
-}
-
-
-uint64_t bench(
-    const char *bench_name,
-    size_t (*fun)(bench_params_t *),
-    size_t (*checkfun)(bench_params_t *, size_t),
-    bench_params_t *params
-) {
-  struct timespec start, end;
-  size_t i, osize = 0, o = 0;
-  size_t time_taken = 0;
-  uint64_t total_repetitions = 0;
-  uint64_t repetitions = 2;
-
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
-
-  while (time_taken < 25 * 1000 * 1000) { // benchmark over at least 1ms
-    if (total_repetitions) {
-      repetitions = total_repetitions; // double previous
-    }
-
-    for (i = 0; i < repetitions; i++) {
-      params->iter = i;
-      if (params->num_ibuf == 1) {
-        params->isample = params->ibuf;
-      } else {
-        params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize));
-      }
-      o = fun(params);
-      if (!o) {
-        fprintf(
-            stderr,
-            "%-19s: %-30s @ lvl %2d: %8ld B: FAILED!\n",
-            params->run_name, bench_name, params->clevel,
-            params->isize);
-        return 0;
-      }
-      osize += o;
-    }
-
-    if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
-
-    time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
-                 (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
-    total_repetitions += repetitions;
-  }
-
-  o = checkfun(params, o);
-  if (!o) {
-    fprintf(
-        stderr,
-        "%-19s: %-30s @ lvl %2d: %8ld B: CHECK FAILED!\n",
-        params->run_name, bench_name, params->clevel,
-        params->isize);
-    return 0;
-  }
-
-  fprintf(
-      stderr,
-      "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
-      params->run_name, bench_name, params->clevel,
-      params->isize, osize / total_repetitions,
-      total_repetitions, time_taken, time_taken / total_repetitions,
-      ((double) 1000 * params->isize * total_repetitions) / time_taken
-  );
-
-  return time_taken;
-}
-
-int main(int argc, char *argv[]) {
-  char *run_name;
-
-  struct stat st;
-  size_t bytes_read;
-
-  const char *dict_fn;
-  size_t dict_size;
-  char *dict_buf;
-  FILE *dict_file;
-
-  const char *in_fn;
-  size_t in_size;
-  size_t num_in_buf;
-  size_t cur_in_buf;
-  char *in_buf;
-  FILE *in_file;
-
-  size_t out_size;
-  char *out_buf;
-
-  size_t check_size;
-  char *check_buf;
-
-  LZ4_stream_t *ctx;
-  LZ4_streamHC_t *hcctx;
-  LZ4F_cctx *cctx;
-  LZ4F_dctx *dctx;
-  LZ4F_CDict *cdict;
-  LZ4F_preferences_t prefs;
-  LZ4F_compressOptions_t options;
-
-  int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
-  unsigned int clevelidx;
-
-  bench_params_t params;
-
-  if (argc != 4) return 1;
-  run_name = argv[1];
-  dict_fn = argv[2];
-  in_fn = argv[3];
-
-  if (stat(dict_fn, &st)) return 1;
-  dict_size = st.st_size;
-  dict_buf = (char *)malloc(dict_size);
-  if (!dict_buf) return 1;
-  dict_file = fopen(dict_fn, "r");
-  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
-  if (bytes_read != dict_size) return 1;
-
-  if (stat(in_fn, &st)) return 1;
-  in_size = st.st_size;
-  num_in_buf = 256 * 1024 * 1024 / in_size;
-  if (num_in_buf == 0) {
-    num_in_buf = 1;
-  }
-
-  in_buf = (char *)malloc(in_size * num_in_buf);
-  if (!in_buf) return 1;
-  in_file = fopen(in_fn, "r");
-  bytes_read = fread(in_buf, 1, in_size, in_file);
-  if (bytes_read != in_size) return 1;
-
-  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
-    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
-  }
-
-  check_size = in_size;
-  check_buf = (char *)malloc(check_size);
-  if (!check_buf) return 1;
-
-  memset(&prefs, 0, sizeof(prefs));
-  prefs.autoFlush = 1;
-  if (in_size < 64 * 1024)
-    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
-  prefs.frameInfo.contentSize = in_size;
-
-  memset(&options, 0, sizeof(options));
-  options.stableSrc = 1;
-
-  out_size = LZ4F_compressFrameBound(in_size, &prefs);
-  if ((size_t)LZ4_compressBound(in_size) > out_size) {
-    out_size = LZ4_compressBound(in_size);
-  }
-  out_buf = (char *)malloc(out_size);
-  if (!out_buf) return 1;
-
-  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
-  if (cctx == NULL) return 1;
-
-  if (LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) return 1;
-  if (cctx == NULL) return 1;
-
-  ctx = LZ4_createStream();
-  if (ctx == NULL) return 1;
-
-  hcctx = LZ4_createStreamHC();
-  if (hcctx == NULL) return 1;
-
-  cdict = LZ4F_createCDict(dict_buf, dict_size);
-  if (!cdict) return 1;
-
-  fprintf(stderr, "dict  size: %zd\n", dict_size);
-  fprintf(stderr, "input size: %zd\n", in_size);
-
-  params.run_name = run_name;
-  params.ctx = ctx;
-  params.hcctx = hcctx;
-  params.cctx = cctx;
-  params.dctx = dctx;
-  params.dictbuf = dict_buf;
-  params.dictsize = dict_size;
-  params.obuf = out_buf;
-  params.osize = out_size;
-  params.ibuf = in_buf;
-  params.isize = in_size;
-  params.num_ibuf = num_in_buf;
-  params.checkbuf = check_buf;
-  params.checksize = check_size;
-  params.clevel = 1;
-  params.cdict = NULL;
-  params.prefs = &prefs;
-  params.options = &options;
-
-  for (clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
-    params.clevel = clevels[clevelidx];
-    params.prefs->compressionLevel = clevels[clevelidx];
-    params.cdict = NULL;
-
-    bench("LZ4_compress_default"         , compress_default    , check_lz4 , &params);
-    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , &params);
-    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , &params);
-    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , &params);
-    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, &params);
-    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, &params);
-
-    params.cdict = cdict;
-
-    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, &params);
-    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, &params);
-  }
-
-  return 0;
-}
-- 
cgit v0.12


From 5ed1463bf4668a0e668679ae1e458aa73aa9a6ec Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 21:00:19 -0400
Subject: Remove Debug Log Statements

---
 lib/lz4frame.c | 21 ---------------------
 lib/lz4hc.c    | 14 +++++---------
 2 files changed, 5 insertions(+), 30 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d4d9397..844c8d1 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -95,18 +95,6 @@ You can contact the author at :
 
 #define LZ4F_STATIC_ASSERT(c)    { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
-#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
-#  include <stdio.h>
-static int g_debuglog_enable = 1;
-#  define DEBUGLOG(l, ...) {                                  \
-                if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
-                    fprintf(stderr, __FILE__ ": ");           \
-                    fprintf(stderr, __VA_ARGS__);             \
-                    fprintf(stderr, " \n");                   \
-            }   }
-#else
-#  define DEBUGLOG(l, ...)      {}    /* disabled */
-#endif
 
 /*-************************************
 *  Basic Types
@@ -469,7 +457,6 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
     const char* dictStart = (const char*)dictBuffer;
     LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
-    DEBUGLOG(4, "LZ4F_createCDict(%p) -> %p", dictBuffer, cdict);
     if (!cdict) return NULL;
     if (dictSize > 64 KB) {
         dictStart += dictSize - 64 KB;
@@ -492,7 +479,6 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 
 void LZ4F_freeCDict(LZ4F_CDict* cdict)
 {
-    DEBUGLOG(4, "LZ4F_freeCDict(%p)", cdict);
     if (cdict==NULL) return;  /* support free on NULL */
     FREEMEM(cdict->dictContent);
     LZ4_freeStream(cdict->fastCtx);
@@ -544,7 +530,6 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 static void LZ4F_applyCDict(void* ctx,
                             const LZ4F_CDict* cdict,
                             int level) {
-    DEBUGLOG(5, "LZ4F_applyCDict(%p, %p)", ctx, cdict);
     if (level < LZ4HC_CLEVEL_MIN) {
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
@@ -571,8 +556,6 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     BYTE* dstPtr = dstStart;
     BYTE* headerStart;
 
-    DEBUGLOG(4, "LZ4F_compressBegin_usingCDict(%p, %p)", cctxPtr, cdict);
-
     if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
     MEM_INIT(&prefNull, 0, sizeof(prefNull));
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
@@ -791,7 +774,6 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
-    DEBUGLOG(4, "LZ4F_compressUpdate(%p, %p, %zd)", cctxPtr, srcBuffer, srcSize);
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
@@ -930,9 +912,6 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize,
     BYTE* dstPtr = dstStart;
 
     size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr);
-
-    DEBUGLOG(4, "LZ4F_compressEnd(%p)", cctxPtr);
-
     if (LZ4F_isError(flushSize)) return flushSize;
     dstPtr += flushSize;
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index a12f298..4126ef8 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -88,7 +88,6 @@ typedef enum { noDictCtx, usingDictCtx } dictCtx_directive;
 **************************************/
 static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 {
-    DEBUGLOG(4, "LZ4HC_clearTables(%p)", hc4);
     MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
 }
@@ -96,7 +95,6 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     uptrval startingOffset = hc4->end - hc4->base;
-    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB) {
         LZ4HC_clearTables(hc4);
         startingOffset = 0;
@@ -830,11 +828,9 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    if (LZ4_streamHCPtr != NULL) {
-        LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
-        LZ4_streamHCPtr->internal_donotuse.base = NULL;
-        LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
-    }
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
+    LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;
 }
 
@@ -881,8 +877,8 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
     }
-    LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4_streamHCPtr->internal_donotuse.compressionLevel);
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
+    LZ4HC_clearTables (ctxPtr);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
     if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;
@@ -897,7 +893,7 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
     DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
-    if (ctxPtr->end - ctxPtr->base - ctxPtr->nextToUpdate >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
+    if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
     ctxPtr->lowLimit  = ctxPtr->dictLimit;
-- 
cgit v0.12


From 2be3905fa4974fd20fea4d9eda5a0030c9d19e93 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 18:41:41 -0400
Subject: Integrate lz4frame_static.h Declarations into lz4frame.h

---
 lib/lz4frame.h        | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++
 lib/lz4frame_static.h | 117 ++----------------------------------------------
 2 files changed, 124 insertions(+), 113 deletions(-)

diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index e80b1a1..bd715bd 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -409,3 +409,123 @@ LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);   /* always su
 #endif
 
 #endif  /* LZ4F_H_09782039843 */
+
+#if defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843)
+#define LZ4F_H_STATIC_09782039843
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* These declarations are not stable and may change in the future. They are
+ * therefore only safe to depend on when the caller is statically linked
+ * against the library. To access their declarations, define
+ * LZ4F_STATIC_LINKING_ONLY.
+ *
+ * There is a further protection mechanism where these symbols aren't published
+ * into shared/dynamic libraries. You can override this behavior and force
+ * them to be published by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. Use at
+ * your own risk.
+ */
+#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS
+#define LZ4FLIB_STATIC_API LZ4FLIB_API
+#else
+#define LZ4FLIB_STATIC_API
+#endif
+
+
+/* ---   Error List   --- */
+#define LZ4F_LIST_ERRORS(ITEM) \
+        ITEM(OK_NoError) \
+        ITEM(ERROR_GENERIC) \
+        ITEM(ERROR_maxBlockSize_invalid) \
+        ITEM(ERROR_blockMode_invalid) \
+        ITEM(ERROR_contentChecksumFlag_invalid) \
+        ITEM(ERROR_compressionLevel_invalid) \
+        ITEM(ERROR_headerVersion_wrong) \
+        ITEM(ERROR_blockChecksum_invalid) \
+        ITEM(ERROR_reservedFlag_set) \
+        ITEM(ERROR_allocation_failed) \
+        ITEM(ERROR_srcSize_tooLarge) \
+        ITEM(ERROR_dstMaxSize_tooSmall) \
+        ITEM(ERROR_frameHeader_incomplete) \
+        ITEM(ERROR_frameType_unknown) \
+        ITEM(ERROR_frameSize_wrong) \
+        ITEM(ERROR_srcPtr_wrong) \
+        ITEM(ERROR_decompressionFailed) \
+        ITEM(ERROR_headerChecksum_invalid) \
+        ITEM(ERROR_contentChecksum_invalid) \
+        ITEM(ERROR_frameDecoding_alreadyStarted) \
+        ITEM(ERROR_maxCode)
+
+#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
+
+/* enum list is exposed, to handle specific errors */
+typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
+
+LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
+
+
+
+/**********************************
+ *  Bulk processing dictionary API
+ *********************************/
+typedef struct LZ4F_CDict_s LZ4F_CDict;
+
+/*! LZ4_createCDict() :
+ *  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ *  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ *  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
+LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
+LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
+
+
+/*! LZ4_compressFrame_usingCDict() :
+ *  Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+ *  cctx must point to a context created by LZ4F_createCompressionContext().
+ *  If cdict==NULL, compress without a dictionary.
+ *  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ *  If this condition is not respected, function will fail (@return an errorCode).
+ *  The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+ *  but it's not recommended, as it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer.
+ *           or an error code if it fails (can be tested using LZ4F_isError()) */
+LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
+    LZ4F_cctx* cctx,
+    void* dst, size_t dstCapacity,
+    const void* src, size_t srcSize,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* preferencesPtr);
+
+
+/*! LZ4F_compressBegin_usingCDict() :
+ *  Inits streaming dictionary compression, and writes the frame header into dstBuffer.
+ *  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * `prefsPtr` is optional : you may provide NULL as argument,
+ *  however, it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer for the header,
+ *           or an error code (which can be tested using LZ4F_isError()) */
+LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
+    LZ4F_cctx* cctx,
+    void* dstBuffer, size_t dstCapacity,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* prefsPtr);
+
+
+/*! LZ4F_decompress_usingDict() :
+ *  Same as LZ4F_decompress(), using a predefined dictionary.
+ *  Dictionary is used "in place", without any preprocessing.
+ *  It must remain accessible throughout the entire frame decoding. */
+LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
+    LZ4F_dctx* dctxPtr,
+    void* dstBuffer, size_t* dstSizePtr,
+    const void* srcBuffer, size_t* srcSizePtr,
+    const void* dict, size_t dictSize,
+    const LZ4F_decompressOptions_t* decompressOptionsPtr);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif  /* defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) */
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index be587e6..925a2c5 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -36,121 +36,12 @@
 #ifndef LZ4FRAME_STATIC_H_0398209384
 #define LZ4FRAME_STATIC_H_0398209384
 
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-/* lz4frame_static.h should be used solely in the context of static linking.
- * It contains definitions which are not stable and may change in the future.
- * Never use it in the context of DLL linking.
- *
- * Defining LZ4F_PUBLISH_STATIC_FUNCTIONS allows one to override this. Use at
- * your own risk.
+/* The declarations that formerly were made here have been merged into
+ * lz4frame.h, protected by the LZ4F_STATIC_LINKING_ONLY macro. Going forward,
+ * it is recommended to simply include that header directly.
  */
-#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS
-#define LZ4FLIB_STATIC_API LZ4FLIB_API
-#else
-#define LZ4FLIB_STATIC_API
-#endif
-
 
-/* ---   Dependency   --- */
+#define LZ4F_STATIC_LINKING_ONLY
 #include "lz4frame.h"
 
-
-/* ---   Error List   --- */
-#define LZ4F_LIST_ERRORS(ITEM) \
-        ITEM(OK_NoError) \
-        ITEM(ERROR_GENERIC) \
-        ITEM(ERROR_maxBlockSize_invalid) \
-        ITEM(ERROR_blockMode_invalid) \
-        ITEM(ERROR_contentChecksumFlag_invalid) \
-        ITEM(ERROR_compressionLevel_invalid) \
-        ITEM(ERROR_headerVersion_wrong) \
-        ITEM(ERROR_blockChecksum_invalid) \
-        ITEM(ERROR_reservedFlag_set) \
-        ITEM(ERROR_allocation_failed) \
-        ITEM(ERROR_srcSize_tooLarge) \
-        ITEM(ERROR_dstMaxSize_tooSmall) \
-        ITEM(ERROR_frameHeader_incomplete) \
-        ITEM(ERROR_frameType_unknown) \
-        ITEM(ERROR_frameSize_wrong) \
-        ITEM(ERROR_srcPtr_wrong) \
-        ITEM(ERROR_decompressionFailed) \
-        ITEM(ERROR_headerChecksum_invalid) \
-        ITEM(ERROR_contentChecksum_invalid) \
-        ITEM(ERROR_frameDecoding_alreadyStarted) \
-        ITEM(ERROR_maxCode)
-
-#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
-
-/* enum list is exposed, to handle specific errors */
-typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
-
-LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
-
-
-
-/**********************************
- *  Bulk processing dictionary API
- *********************************/
-typedef struct LZ4F_CDict_s LZ4F_CDict;
-
-/*! LZ4_createCDict() :
- *  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
- *  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
- *  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
-LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
-LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
-
-
-/*! LZ4_compressFrame_usingCDict() :
- *  Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
- *  cctx must point to a context created by LZ4F_createCompressionContext().
- *  If cdict==NULL, compress without a dictionary.
- *  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
- *  If this condition is not respected, function will fail (@return an errorCode).
- *  The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
- *  but it's not recommended, as it's the only way to provide dictID in the frame header.
- * @return : number of bytes written into dstBuffer.
- *           or an error code if it fails (can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
-    LZ4F_cctx* cctx,
-    void* dst, size_t dstCapacity,
-    const void* src, size_t srcSize,
-    const LZ4F_CDict* cdict,
-    const LZ4F_preferences_t* preferencesPtr);
-
-
-/*! LZ4F_compressBegin_usingCDict() :
- *  Inits streaming dictionary compression, and writes the frame header into dstBuffer.
- *  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
- * `prefsPtr` is optional : you may provide NULL as argument,
- *  however, it's the only way to provide dictID in the frame header.
- * @return : number of bytes written into dstBuffer for the header,
- *           or an error code (which can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
-    LZ4F_cctx* cctx,
-    void* dstBuffer, size_t dstCapacity,
-    const LZ4F_CDict* cdict,
-    const LZ4F_preferences_t* prefsPtr);
-
-
-/*! LZ4F_decompress_usingDict() :
- *  Same as LZ4F_decompress(), using a predefined dictionary.
- *  Dictionary is used "in place", without any preprocessing.
- *  It must remain accessible throughout the entire frame decoding. */
-LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
-    LZ4F_dctx* dctxPtr,
-    void* dstBuffer, size_t* dstSizePtr,
-    const void* srcBuffer, size_t* srcSizePtr,
-    const void* dict, size_t dictSize,
-    const LZ4F_decompressOptions_t* decompressOptionsPtr);
-
-
-#if defined (__cplusplus)
-}
-#endif
-
 #endif /* LZ4FRAME_STATIC_H_0398209384 */
-- 
cgit v0.12


From 2dfc7cbe826342905faa479175f6f050e8445aff Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 20:55:38 -0400
Subject: Change Over Includes in the Project

---
 lib/lz4frame.c    | 3 ++-
 programs/lz4io.c  | 2 +-
 tests/frametest.c | 3 ++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 844c8d1..b616463 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -73,7 +73,8 @@ You can contact the author at :
 /*-************************************
 *  Includes
 **************************************/
-#include "lz4frame_static.h"
+#define LZ4F_STATIC_LINKING_ONLY
+#include "lz4frame.h"
 #define LZ4_STATIC_LINKING_ONLY
 #include "lz4.h"
 #define LZ4_HC_STATIC_LINKING_ONLY
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 6d0d0d0..ccf4fa1 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -56,8 +56,8 @@
 #include "lz4io.h"
 #include "lz4.h"       /* still required for legacy format */
 #include "lz4hc.h"     /* still required for legacy format */
+#define LZ4F_STATIC_LINKING_ONLY
 #include "lz4frame.h"
-#include "lz4frame_static.h"
 
 
 /*****************************
diff --git a/tests/frametest.c b/tests/frametest.c
index 74d9c88..01f1300 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -41,7 +41,8 @@
 #include <string.h>     /* strcmp */
 #include <time.h>       /* clock_t, clock(), CLOCKS_PER_SEC */
 #include <assert.h>
-#include "lz4frame_static.h"
+#define LZ4F_STATIC_LINKING_ONLY
+#include "lz4frame.h"
 #include "lz4.h"        /* LZ4_VERSION_STRING */
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"     /* XXH64 */
-- 
cgit v0.12


From 27c6eec18d38d78f058e87e415ac979129342861 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 24 Apr 2018 18:50:03 -0400
Subject: Multiply-Include Header to Check Guard Macro Correctness

---
 tests/frametest.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/frametest.c b/tests/frametest.c
index 01f1300..7d69ff7 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -41,8 +41,11 @@
 #include <string.h>     /* strcmp */
 #include <time.h>       /* clock_t, clock(), CLOCKS_PER_SEC */
 #include <assert.h>
+#include "lz4frame.h"   /* include multiple times to test correctness/safety */
+#include "lz4frame.h"
 #define LZ4F_STATIC_LINKING_ONLY
 #include "lz4frame.h"
+#include "lz4frame.h"
 #include "lz4.h"        /* LZ4_VERSION_STRING */
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"     /* XXH64 */
-- 
cgit v0.12


From ff9b4cf82678f9643d256129d06098b692072584 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Wed, 25 Apr 2018 01:40:12 +0300
Subject: lz4_Block_format.md: clarify on short inputs and restrictions

It occurred to me that the formula "The last 5 bytes are always
literals", on the list of "assumptions made by the decoder", is
remarkably ambiguous.  Suppose the decoder is presented with 5 bytes.
Are they literals?  It may seem that the decoder degenerates
to memcpy on short inputs.  But of course the answer is no,
so the formula needs some clarification.

Parsing restrictions should be explained as well, otherwise they look
like arbitrary numbers.  The 5-byte restriction has been mentioned
recently in connection with the shortcut in LZ4_decompress_generic,
so I add that.  The second restriction is left to be explained
by the author.

I also took the liberty to explain that empty inputs "are either
unrepresentable or can be represented with a null byte".  This wording
may actually have some merit: it leaves for the implementation,
as opposed to the spec, to decide whether the encoder can compress
empty inputs, and whether the decoder can produce an empty output
(which the implementation should further clarify).
---
 doc/lz4_Block_format.md | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/doc/lz4_Block_format.md b/doc/lz4_Block_format.md
index 4e39b41..dd4c91b 100644
--- a/doc/lz4_Block_format.md
+++ b/doc/lz4_Block_format.md
@@ -109,15 +109,24 @@ Parsing restrictions
 There are specific parsing rules to respect in order to remain compatible
 with assumptions made by the decoder :
 
-1. The last 5 bytes are always literals
+1. The last 5 bytes are always literals.  In other words, the last five bytes
+   from the uncompressed input (or all bytes, if the input has less than five
+   bytes) must be encoded as literals on behalf of the last sequence.
+   The last sequence is incomplete, and stops right after the literals.
 2. The last match must start at least 12 bytes before end of block.
    Consequently, a block with less than 13 bytes cannot be compressed.
 
 These rules are in place to ensure that the decoder
 will never read beyond the input buffer, nor write beyond the output buffer.
 
-Note that the last sequence is also incomplete,
-and stops right after literals.
+1. To copy literals from a non-last sequence, an 8-byte copy instruction
+   can always be safely issued (without reading past the input), because
+   the literals are followed by a 2-byte offset, and the last sequence
+   is at least 1+5 bytes long.
+2. TODO: explain the benefits of the second restriction.
+
+Empty inputs are either unrepresentable or can be represented with a null byte,
+which can be interpreted as a token without literals and without a match.
 
 
 Additional notes
-- 
cgit v0.12


From b4eda8d08f307dfc3cbd0d06baea4e6c581b70de Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Mon, 23 Apr 2018 08:37:44 +0300
Subject: lz4.c: refactor the decoding routines

I noticed that LZ4_decompress_generic is sometimes instantiated with
identical set of parameters, or (what's worse) with a subtly different
sets of parameters.  For example, LZ4_decompress_fast_withPrefix64k is
instantiated as follows:

    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize,
		full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);

while the equivalent withPrefix64k call in LZ4_decompress_usingDict_generic
passes 0 for the last argument instead of 64 KB.  It turns out that there
is no difference in this case: if you change 64 KB to 0 KB in
LZ4_decompress_fast_withPrefix64k, you get the same binary code.

Moreover, because it's been clarified that LZ4_decompress_fast doesn't
check match offsets, it is now obvious that both of these fast/withPrefix64k
instantiations are simply redundant.  Exactly because LZ4_decompress_fast
doesn't check offsets, it serves well with any prefixed dictionary.

There's a difference, though, with LZ4_decompress_safe_withPrefix64k.
It also passes 64 KB as the last argument, and if you change that to 0,
as in LZ4_decompress_usingDict_generic, you get a completely different
binary code.  It seems that passing 0 enables offset checking:

    const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));

However, the resulting code seems to run a bit faster.  How come
enabling extra checks can make the code run faster?  Curiouser and
curiouser!  This needs extra study.  Currently I take the view that
the dictSize should be set to non-zero when nothing else will do,
i.e. when passing the external dictionary via dictStart.  Otherwise,
lowPrefix betrays just enough information about the dictionary.

    * * *

Anyway, with this change, I instantiate all the necessary cases as
functions with distinctive names, which also take fewer arguments and
are therefore less error-prone.  I also make the functions non-inline.
(The compiler won't inline the functions because they are used more than
once.  Hence I attach LZ4_FORCE_O2_GCC_PPC64LE to the instances while
removing from the callers.)  The number of instances is now is reduced
from 18 (safe+fast+partial+4*continue+4*prefix+4*dict+2*prefix64+forceExtDict)
down to 7 (safe+fast+partial+2*prefix+2*dict).  The size of the code is
not the only issue here.  Separate helper function are much more
amenable to profile-guided optimization: it is enough to profile only
a few basic functions, while the other less-often used functions, such
as LZ4_decompress_*_continue, will benefit automatically.

This is the list of LZ4_decompress* functions in liblz4.so, sorted by size.
Exported functions are marked with a capital T.

$ nm -S lib/liblz4.so |grep -wi T |grep LZ4_decompress |sort -k2
0000000000016260 0000000000000005 T LZ4_decompress_fast_withPrefix64k
0000000000016dc0 0000000000000025 T LZ4_decompress_fast_usingDict
0000000000016d80 0000000000000040 T LZ4_decompress_safe_usingDict
0000000000016d10 000000000000006b T LZ4_decompress_fast_continue
0000000000016c70 000000000000009f T LZ4_decompress_safe_continue
00000000000156c0 000000000000059c T LZ4_decompress_fast
0000000000014a90 00000000000005fa T LZ4_decompress_safe
0000000000015c60 00000000000005fa T LZ4_decompress_safe_withPrefix64k
0000000000002280 00000000000005fa t LZ4_decompress_safe_withSmallPrefix
0000000000015090 000000000000062f T LZ4_decompress_safe_partial
0000000000002880 00000000000008ea t LZ4_decompress_fast_extDict
0000000000016270 0000000000000993 t LZ4_decompress_safe_forceExtDict
---
 lib/lz4.c | 132 +++++++++++++++++++++++++++++++++++++-------------------------
 1 file changed, 79 insertions(+), 53 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index bb6b619..2fad34f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -92,6 +92,7 @@
 *  Dependency
 **************************************/
 #define LZ4_STATIC_LINKING_ONLY
+#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */
 #include "lz4.h"
 /* see also "memory routines" below */
 
@@ -1666,6 +1667,8 @@ _output_error:
 }
 
 
+/*===== Instantiate the API decoding functions. =====*/
+
 LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
 {
@@ -1687,9 +1690,63 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
 {
     return LZ4_decompress_generic(source, dest, 0, originalSize,
                                   endOnOutputSize, full, 0, withPrefix64k,
-                                  (BYTE*)(dest - 64 KB), NULL, 64 KB);
+                                  (BYTE*)dest - 64 KB, NULL, 0);
+}
+
+/*===== Instantiate a few more decoding cases, used more than once. =====*/
+
+LZ4_FORCE_O2_GCC_PPC64LE /* Exported, an obsolete API function. */
+int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
+{
+    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
+                                  endOnInputSize, full, 0, withPrefix64k,
+                                  (BYTE*)dest - 64 KB, NULL, 0);
+}
+
+/* Another obsolete API function, paired with the previous one. */
+int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
+{
+    /* LZ4_decompress_fast doesn't validate match offsets,
+     * and thus serves well with any prefixed dictionary. */
+    return LZ4_decompress_fast(source, dest, originalSize);
+}
+
+LZ4_FORCE_O2_GCC_PPC64LE
+static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize,
+                                               size_t dictSize)
+{
+    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
+                                  endOnInputSize, full, 0, noDict,
+                                  (BYTE*)dest-dictSize, NULL, 0);
 }
 
+LZ4_FORCE_INLINE
+int LZ4_decompress_safe_withPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize,
+                                   size_t dictSize)
+{
+    if (dictSize >= 64 KB - 1)
+        return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
+    return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
+}
+
+LZ4_FORCE_O2_GCC_PPC64LE /* Exported under another name, for tests/fullbench.c */
+#define LZ4_decompress_safe_extDict LZ4_decompress_safe_forceExtDict
+int LZ4_decompress_safe_extDict(const char* source, char* dest, int compressedSize, int maxOutputSize,
+                                const char* dictStart, size_t dictSize)
+{
+    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
+                                  endOnInputSize, full, 0, usingExtDict,
+                                  (BYTE*)dest, (const BYTE*)dictStart, dictSize);
+}
+
+LZ4_FORCE_O2_GCC_PPC64LE
+static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize,
+                                       const char* dictStart, size_t dictSize)
+{
+    return LZ4_decompress_generic(source, dest, 0, originalSize,
+                                  endOnOutputSize, full, 0, usingExtDict,
+                                  (BYTE*)dest, (const BYTE*)dictStart, dictSize);
+}
 
 /*===== streaming decompression functions =====*/
 
@@ -1730,25 +1787,26 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
     If it's not possible, save the relevant part of decoded data into a safe buffer,
     and indicate where it stands using LZ4_setStreamDecode()
 */
-LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
 {
     LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
     int result;
 
-    if (lz4sd->prefixEnd == (BYTE*)dest) {
-        result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
-                                        endOnInputSize, full, 0,
-                                        usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
+    if (lz4sd->prefixSize == 0) {
+        result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);
+        if (result <= 0) return result;
+        lz4sd->prefixSize = result;
+        lz4sd->prefixEnd = (BYTE*)dest + result;
+    } else if (lz4sd->prefixEnd == (BYTE*)dest) {
+        result = LZ4_decompress_safe_withPrefix(source, dest, compressedSize, maxOutputSize, lz4sd->prefixSize);
         if (result <= 0) return result;
         lz4sd->prefixSize += result;
         lz4sd->prefixEnd  += result;
     } else {
         lz4sd->extDictSize = lz4sd->prefixSize;
         lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
-        result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
-                                        endOnInputSize, full, 0,
-                                        usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize);
+        result = LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize,
+                                             (const char*)lz4sd->externalDict, lz4sd->extDictSize);
         if (result <= 0) return result;
         lz4sd->prefixSize = result;
         lz4sd->prefixEnd  = (BYTE*)dest + result;
@@ -1757,25 +1815,21 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
     return result;
 }
 
-LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)
 {
     LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
     int result;
 
-    if (lz4sd->prefixEnd == (BYTE*)dest) {
-        result = LZ4_decompress_generic(source, dest, 0, originalSize,
-                                        endOnOutputSize, full, 0,
-                                        usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
+    if (lz4sd->prefixSize == 0 || lz4sd->prefixEnd == (BYTE*)dest) {
+        result = LZ4_decompress_fast(source, dest, originalSize);
         if (result <= 0) return result;
         lz4sd->prefixSize += originalSize;
         lz4sd->prefixEnd  += originalSize;
     } else {
         lz4sd->extDictSize = lz4sd->prefixSize;
         lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
-        result = LZ4_decompress_generic(source, dest, 0, originalSize,
-                                        endOnOutputSize, full, 0,
-                                        usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize);
+        result = LZ4_decompress_fast_extDict(source, dest, originalSize,
+                                             (const char*)lz4sd->externalDict, lz4sd->extDictSize);
         if (result <= 0) return result;
         lz4sd->prefixSize = originalSize;
         lz4sd->prefixEnd  = (BYTE*)dest + originalSize;
@@ -1792,36 +1846,20 @@ Advanced decoding functions :
     the dictionary must be explicitly provided within parameters
 */
 
-LZ4_FORCE_O2_GCC_PPC64LE
-LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
-{
-    if (dictSize==0)
-        return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0);
-    if (dictStart+dictSize == dest) {
-        if (dictSize >= (int)(64 KB - 1))
-            return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0);
-        return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0);
-    }
-    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
-}
-
-LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
 {
-    return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize);
+    if (dictSize==0)
+        return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);
+    if (dictStart+dictSize == dest)
+        return LZ4_decompress_safe_withPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
+    return LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize);
 }
 
-LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
 {
-    return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize);
-}
-
-/* debug function */
-LZ4_FORCE_O2_GCC_PPC64LE
-int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
-{
-    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
+    if (dictSize==0 || dictStart+dictSize == dest)
+        return LZ4_decompress_fast(source, dest, originalSize);
+    return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, dictSize);
 }
 
 
@@ -1892,16 +1930,4 @@ char* LZ4_slideInputBuffer (void* state)
     return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary;
 }
 
-/* Obsolete streaming decompression functions */
-
-int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
-{
-    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
-}
-
-int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
-{
-    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
-}
-
 #endif   /* LZ4_COMMONDEFS_ONLY */
-- 
cgit v0.12


From bd92689798292f8ab8d2b48f31cd4b49bfa6d87b Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Wed, 25 Apr 2018 06:42:57 -0700
Subject: minor edit of block format

clarifying parsing restrictions near end of block.
---
 doc/lz4_Block_format.md | 60 ++++++++++++++++++++++++++-----------------------
 1 file changed, 32 insertions(+), 28 deletions(-)

diff --git a/doc/lz4_Block_format.md b/doc/lz4_Block_format.md
index dd4c91b..5438730 100644
--- a/doc/lz4_Block_format.md
+++ b/doc/lz4_Block_format.md
@@ -1,6 +1,6 @@
 LZ4 Block Format Description
 ============================
-Last revised: 2015-05-07.
+Last revised: 2018-04-25.
 Author : Yann Collet
 
 
@@ -29,8 +29,8 @@ An LZ4 compressed block is composed of sequences.
 A sequence is a suite of literals (not-compressed bytes),
 followed by a match copy.
 
-Each sequence starts with a token.
-The token is a one byte value, separated into two 4-bits fields.
+Each sequence starts with a `token`.
+The `token` is a one byte value, separated into two 4-bits fields.
 Therefore each field ranges from 0 to 15.
 
 
@@ -42,46 +42,46 @@ If it is 15, then we need to add some more bytes to indicate the full length.
 Each additional byte then represent a value from 0 to 255,
 which is added to the previous value to produce a total length.
 When the byte value is 255, another byte is output.
-There can be any number of bytes following the token. There is no "size limit".
+There can be any number of bytes following `token`. There is no "size limit".
 (Side note : this is why a not-compressible input block is expanded by 0.4%).
 
-Example 1 : A length of 48 will be represented as :
+Example 1 : A literal length of 48 will be represented as :
 
   - 15 : value for the 4-bits High field
   - 33 : (=48-15) remaining length to reach 48
 
-Example 2 : A length of 280 will be represented as :
+Example 2 : A literal length of 280 will be represented as :
 
   - 15  : value for the 4-bits High field
   - 255 : following byte is maxed, since 280-15 >= 255
   - 10  : (=280 - 15 - 255) ) remaining length to reach 280
 
-Example 3 : A length of 15 will be represented as :
+Example 3 : A literal length of 15 will be represented as :
 
   - 15 : value for the 4-bits High field
   - 0  : (=15-15) yes, the zero must be output
 
-Following the token and optional length bytes, are the literals themselves.
+Following `token` and optional length bytes, are the literals themselves.
 They are exactly as numerous as previously decoded (length of literals).
 It's possible that there are zero literal.
 
 
 Following the literals is the match copy operation.
 
-It starts by the offset.
+It starts by the `offset`.
 This is a 2 bytes value, in little endian format
 (the 1st byte is the "low" byte, the 2nd one is the "high" byte).
 
-The offset represents the position of the match to be copied from.
+The `offset` represents the position of the match to be copied from.
 1 means "current position - 1 byte".
-The maximum offset value is 65535, 65536 cannot be coded.
+The maximum `offset` value is 65535, 65536 cannot be coded.
 Note that 0 is an invalid value, not used.
 
-Then we need to extract the match length.
+Then we need to extract the `matchlength`.
 For this, we use the second token field, the low 4-bits.
 Value, obviously, ranges from 0 to 15.
 However here, 0 means that the copy operation will be minimal.
-The minimum length of a match, called minmatch, is 4.
+The minimum length of a match, called `minmatch`, is 4.
 As a consequence, a 0 value means 4 bytes, and a value of 15 means 19+ bytes.
 Similar to literal length, on reaching the highest possible value (15),
 we output additional bytes, one at a time, with values ranging from 0 to 255.
@@ -90,18 +90,18 @@ A 255 value means there is another byte to read and add.
 There is no limit to the number of optional bytes that can be output this way.
 (This points towards a maximum achievable compression ratio of about 250).
 
-Decoding the matchlength reaches the end of current sequence.
+Decoding the `matchlength` reaches the end of current sequence.
 Next byte will be the start of another sequence.
 But before moving to next sequence,
 it's time to use the decoded match position and length.
-The decoder copies matchlength bytes from match position to current position.
+The decoder copies `matchlength` bytes from match position to current position.
 
-In some cases, matchlength is larger than offset.
-Therefore, match pos + match length > current pos,
+In some cases, `matchlength` is larger than `offset`.
+Therefore, `match_pos + matchlength > current_pos`,
 which means that later bytes to copy are not yet decoded.
 This is called an "overlap match", and must be handled with special care.
-The most common case is an offset of 1,
-meaning the last byte is repeated matchlength times.
+A common case is an offset of 1,
+meaning the last byte is repeated `matchlength` times.
 
 
 Parsing restrictions
@@ -114,19 +114,23 @@ with assumptions made by the decoder :
    bytes) must be encoded as literals on behalf of the last sequence.
    The last sequence is incomplete, and stops right after the literals.
 2. The last match must start at least 12 bytes before end of block.
-   Consequently, a block with less than 13 bytes cannot be compressed.
+   The last match is part of the penultimate sequence,
+   since the last sequence stops right after literals.
+   Note that, as a consequence, blocks < 13 bytes cannot be compressed.
 
 These rules are in place to ensure that the decoder
-will never read beyond the input buffer, nor write beyond the output buffer.
+can speculatively execute copy instructions
+without ever reading nor writing beyond provided I/O buffers.
 
 1. To copy literals from a non-last sequence, an 8-byte copy instruction
-   can always be safely issued (without reading past the input), because
-   the literals are followed by a 2-byte offset, and the last sequence
-   is at least 1+5 bytes long.
-2. TODO: explain the benefits of the second restriction.
-
-Empty inputs are either unrepresentable or can be represented with a null byte,
-which can be interpreted as a token without literals and without a match.
+   can always be safely issued (without reading past the input),
+   because literals are followed by a 2-byte offset,
+   and last sequence is at least 1+5 bytes long.
+2. Similarly, a match operation can speculatively copy up to 12 bytes
+   while remaining within output buffer boundaries.
+
+Empty inputs can be represented with a zero byte,
+interpreted as a token without literals and without a match.
 
 
 Additional notes
-- 
cgit v0.12


From 5603d30f81c99fbe55f48d953e3df728720ff982 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Thu, 26 Apr 2018 07:46:26 +0300
Subject: lz4.c: fixed the LZ4_decompress_safe_continue case

The previous change broke decoding with a ring buffer.  That's because
I didn't realize that the "double dictionary mode" was possible, i.e.
that the decoding routine can look both at the first part of the
dictionary passed as prefix and the second part passed via dictStart+dictSize.

So this change introduces the LZ4_decompress_safe_doubleDict helper,
which handles this "double dictionary" situation.  (This is a bit of
a misnomer, there is only one dictionary, but I can't think of a better
name, and perhaps the designation is not all too bad.)  The helper is
used only once, in LZ4_decompress_safe_continue, it should be inlined
with LZ4_FORCE_O2_GCC_PPC64LE attached to LZ4_decompress_safe_continue.

(Also, in the helper functions, I change the dictStart parameter type
to "const void*", to avoid a cast when calling helpers.  In the helpers,
the upcast to "BYTE*" is still required, for compatibility with C++.)

So this fixes the case of LZ4_decompress_safe_continue, and I'm
surprised by the fact that the fuzzer is now happy and does not detect
a similar problem with LZ4_decompress_fast_continue.  So before fixing
LZ4_decompress_fast_continue, the next logical step is to enhance
the fuzzer.
---
 lib/lz4.c         | 53 ++++++++++++++++++++++++++++++++++++-----------------
 tests/fullbench.c |  2 +-
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 2fad34f..eb3da21 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1713,26 +1713,17 @@ int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int origin
 
 LZ4_FORCE_O2_GCC_PPC64LE
 static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize,
-                                               size_t dictSize)
+                                               size_t prefixSize)
 {
     return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
                                   endOnInputSize, full, 0, noDict,
-                                  (BYTE*)dest-dictSize, NULL, 0);
-}
-
-LZ4_FORCE_INLINE
-int LZ4_decompress_safe_withPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize,
-                                   size_t dictSize)
-{
-    if (dictSize >= 64 KB - 1)
-        return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
-    return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
+                                  (BYTE*)dest-prefixSize, NULL, 0);
 }
 
 LZ4_FORCE_O2_GCC_PPC64LE /* Exported under another name, for tests/fullbench.c */
 #define LZ4_decompress_safe_extDict LZ4_decompress_safe_forceExtDict
 int LZ4_decompress_safe_extDict(const char* source, char* dest, int compressedSize, int maxOutputSize,
-                                const char* dictStart, size_t dictSize)
+                                const void* dictStart, size_t dictSize)
 {
     return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
                                   endOnInputSize, full, 0, usingExtDict,
@@ -1741,13 +1732,26 @@ int LZ4_decompress_safe_extDict(const char* source, char* dest, int compressedSi
 
 LZ4_FORCE_O2_GCC_PPC64LE
 static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize,
-                                       const char* dictStart, size_t dictSize)
+                                       const void* dictStart, size_t dictSize)
 {
     return LZ4_decompress_generic(source, dest, 0, originalSize,
                                   endOnOutputSize, full, 0, usingExtDict,
                                   (BYTE*)dest, (const BYTE*)dictStart, dictSize);
 }
 
+/* The "double dictionary" mode, for use with e.g. ring buffers: the first part
+ * of the dictionary is passed as prefix, and the second via dictStart + dictSize.
+ * These routines are used only once, in LZ4_decompress_*_continue().
+ */
+LZ4_FORCE_INLINE
+int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize,
+                                   size_t prefixSize, const void* dictStart, size_t dictSize)
+{
+    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
+                                  endOnInputSize, full, 0, usingExtDict,
+                                  (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
+}
+
 /*===== streaming decompression functions =====*/
 
 LZ4_streamDecode_t* LZ4_createStreamDecode(void)
@@ -1787,26 +1791,38 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
     If it's not possible, save the relevant part of decoded data into a safe buffer,
     and indicate where it stands using LZ4_setStreamDecode()
 */
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
 {
     LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
     int result;
 
     if (lz4sd->prefixSize == 0) {
+        /* The first call, no dictionary yet. */
+        assert(lz4sd->extDictSize == 0);
         result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);
         if (result <= 0) return result;
         lz4sd->prefixSize = result;
         lz4sd->prefixEnd = (BYTE*)dest + result;
     } else if (lz4sd->prefixEnd == (BYTE*)dest) {
-        result = LZ4_decompress_safe_withPrefix(source, dest, compressedSize, maxOutputSize, lz4sd->prefixSize);
+        /* They're rolling the current segment. */
+        if (lz4sd->prefixSize >= 64 KB - 1)
+            result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
+        else if (lz4sd->extDictSize == 0)
+            result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize,
+                                                         lz4sd->prefixSize);
+        else
+            result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize,
+                                                    lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
         if (result <= 0) return result;
         lz4sd->prefixSize += result;
         lz4sd->prefixEnd  += result;
     } else {
+        /* The buffer wraps around, or they're switching to another buffer. */
         lz4sd->extDictSize = lz4sd->prefixSize;
         lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
         result = LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize,
-                                             (const char*)lz4sd->externalDict, lz4sd->extDictSize);
+                                             lz4sd->externalDict, lz4sd->extDictSize);
         if (result <= 0) return result;
         lz4sd->prefixSize = result;
         lz4sd->prefixEnd  = (BYTE*)dest + result;
@@ -1850,8 +1866,11 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed
 {
     if (dictSize==0)
         return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);
-    if (dictStart+dictSize == dest)
-        return LZ4_decompress_safe_withPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
+    if (dictStart+dictSize == dest) {
+        if (dictSize >= 64 KB - 1)
+            return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
+        return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
+    }
     return LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize);
 }
 
diff --git a/tests/fullbench.c b/tests/fullbench.c
index f489392..1939aeb 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -277,7 +277,7 @@ static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int in
 }
 
 #ifndef LZ4_DLL_IMPORT
-extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const char* dict, int dictSize);
+extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize);
 
 static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize)
 {
-- 
cgit v0.12


From 1148173c5dd1ad9b672c63fd0da110e3c2d66274 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 26 Apr 2018 13:01:59 -0700
Subject: introduced ability to parse for decompression speed

triggered through an enum.

Now, it's still necessary to properly expose this capability
all the way up to the cli.
---
 lib/lz4hc.c | 53 ++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 4126ef8..3593da7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -199,6 +199,7 @@ LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
 }
 
 typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e;
+typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e;
 
 LZ4_FORCE_INLINE int
 LZ4HC_InsertAndGetWiderMatch (
@@ -211,7 +212,8 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE** startpos,
     const int maxNbAttempts,
     const int patternAnalysis,
-    const dictCtx_directive dict)
+    const dictCtx_directive dict,
+    const HCfavor_e favorDecSpeed)
 {
     U16* const chainTable = hc4->chainTable;
     U32* const HashTable = hc4->hashTable;
@@ -240,7 +242,10 @@ LZ4HC_InsertAndGetWiderMatch (
     while ((matchIndex>=lowLimit) && (nbAttempts)) {
         DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
         nbAttempts--;
-        if (matchIndex >= dictLimit) {
+        assert(matchIndex < ipIndex);
+        if (favorDecSpeed && (ipIndex - matchIndex < 8)) {
+            /* do nothing */
+        } else if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
             assert(longest >= 1);
             if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - delta + longest - 1)) {
@@ -326,14 +331,12 @@ LZ4HC_InsertAndGetWiderMatch (
                 }
             }
 
-            {
-                U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex);
+            {   U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex);
                 dictMatchIndex -= nextOffset;
                 matchIndex -= nextOffset;
             }
         }
     }
-
     return longest;
 }
 
@@ -349,7 +352,7 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict);
+    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
 }
 
 
@@ -484,7 +487,7 @@ _Search2:
         if (ip+ml <= mflimit)
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
-                            maxNbAttempts, patternAnalysis, dict);
+                            maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
         else
             ml2 = ml;
 
@@ -531,7 +534,7 @@ _Search3:
         if (start2 + ml2 <= mflimit)
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
-                            maxNbAttempts, patternAnalysis, dict);
+                            maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
         else
             ml3 = ml2;
 
@@ -651,12 +654,14 @@ _dest_overflow:
     return 0;
 }
 
+
 static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx,
     const char* const source, char* dst,
     int* srcSizePtr, int dstCapacity,
     int const nbSearches, size_t sufficient_len,
     const limitedOutput_directive limit, int const fullUpdate,
-    const dictCtx_directive dict);
+    const dictCtx_directive dict,
+    HCfavor_e favorDecSpeed);
 
 
 LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
@@ -711,7 +716,9 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
         return LZ4HC_compress_optimal(ctx,
                             src, dst, srcSizePtr, dstCapacity,
                             cParam.nbSearches, cParam.targetLength, limit,
-                            cLevel == LZ4HC_CLEVEL_MAX, dict);  /* ultra mode */
+                            cLevel == LZ4HC_CLEVEL_MAX,   /* ultra mode */
+                            dict,
+                            favorDecompressionSpeed);
     }
 }
 
@@ -1082,23 +1089,26 @@ LZ4_FORCE_INLINE LZ4HC_match_t
 LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
                       const BYTE* ip, const BYTE* const iHighLimit,
                       int minLen, int nbSearches,
-                      const dictCtx_directive dict)
+                      const dictCtx_directive dict,
+                      const HCfavor_e favorDecSpeed)
 {
     LZ4HC_match_t match = { 0 , 0 };
     const BYTE* matchPtr = NULL;
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
+    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
                                 ip, ip, iHighLimit, minLen, &matchPtr, &ip,
-                                nbSearches, 1 /* patternAnalysis */, dict);
+                                nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
     if (matchLength <= minLen) return match;
+    if (favorDecSpeed) {
+        if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */
+    }
     match.len = matchLength;
     match.off = (int)(ip-matchPtr);
     return match;
 }
 
-
 static int LZ4HC_compress_optimal (
     LZ4HC_CCtx_internal* ctx,
     const char* const source,
@@ -1109,7 +1119,8 @@ static int LZ4HC_compress_optimal (
     size_t sufficient_len,
     const limitedOutput_directive limit,
     int const fullUpdate,
-    const dictCtx_directive dict
+    const dictCtx_directive dict,
+    HCfavor_e favorDecSpeed
     )
 {
 #define TRAILING_LITERALS 3
@@ -1125,6 +1136,7 @@ static int LZ4HC_compress_optimal (
     BYTE* oend = op + dstCapacity;
 
     /* init */
+    favorDecSpeed = favorCompressionRatio;
     DEBUGLOG(5, "LZ4HC_compress_optimal");
     *srcSizePtr = 0;
     if (limit == limitedDestSize) oend -= LASTLITERALS;   /* Hack for support LZ4 format restriction */
@@ -1137,7 +1149,7 @@ static int LZ4HC_compress_optimal (
          int best_mlen, best_off;
          int cur, last_match_pos = 0;
 
-         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict);
+         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed);
          if (firstMatch.len==0) { ip++; continue; }
 
          if ((size_t)firstMatch.len > sufficient_len) {
@@ -1207,10 +1219,10 @@ static int LZ4HC_compress_optimal (
 
              DEBUGLOG(7, "search at rPos:%u", cur);
              if (fullUpdate)
-                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict);
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed);
              else
                  /* only test matches of minimum length; slightly faster, but misses a few bytes */
-                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict);
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict, favorDecSpeed);
              if (!newMatch.len) continue;
 
              if ( ((size_t)newMatch.len > sufficient_len)
@@ -1258,7 +1270,10 @@ static int LZ4HC_compress_optimal (
                          price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
                      }
 
-                     if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) {
+                    assert(opt[pos].price > 1);
+                    assert((U32)favorDecSpeed <= 1);
+                     if (pos > last_match_pos+TRAILING_LITERALS
+                      || price <= opt[pos].price - (int)favorDecSpeed) {
                          DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
                                      pos, price, ml);
                          assert(pos < LZ4_OPT_NUM);
-- 
cgit v0.12


From a2edeac201a7c1c7869d3754cd4dd5d49997357e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 26 Apr 2018 16:53:40 -0400
Subject: Limit Dictionary Size During LZ4F Decompression

Fixes lz4/lz4#517.
---
 lib/lz4frame.c | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index b616463..4d6d39c 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1502,11 +1502,19 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             }   }
 
             if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {
+                const char *dict = (const char *)dctx->dict;
+                size_t dictSize = dctx->dictSize;
+                int decodedSize;
+                if (dict && dictSize > 1 GB) {
+                    /* the dictSize param is an int, avoid truncation / sign issues */
+                    dict += dictSize - 1 GB;
+                    dictSize = 1 GB;
+                }
                 /* enough capacity in `dst` to decompress directly there */
-                int const decodedSize = LZ4_decompress_safe_usingDict(
+                decodedSize = LZ4_decompress_safe_usingDict(
                         (const char*)selectedIn, (char*)dstPtr,
                         (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
-                        (const char*)dctx->dict, (int)dctx->dictSize);
+                        dict, (int)dictSize);
                 if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC);   /* decompression failed */
                 if (dctx->frameInfo.contentChecksumFlag)
                     XXH32_update(&(dctx->xxh), dstPtr, decodedSize);
@@ -1538,10 +1546,19 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             }
 
             /* Decode block */
-            {   int const decodedSize = LZ4_decompress_safe_usingDict(
+            {
+                const char *dict = (const char *)dctx->dict;
+                size_t dictSize = dctx->dictSize;
+                int decodedSize;
+                if (dict && dictSize > 1 GB) {
+                    /* the dictSize param is an int, avoid truncation / sign issues */
+                    dict += dictSize - 1 GB;
+                    dictSize = 1 GB;
+                }
+                decodedSize = LZ4_decompress_safe_usingDict(
                         (const char*)selectedIn, (char*)dctx->tmpOut,
                         (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
-                        (const char*)dctx->dict, (int)dctx->dictSize);
+                        dict, (int)dictSize);
                 if (decodedSize < 0)  /* decompression failed */
                     return err0r(LZ4F_ERROR_decompressionFailed);
                 if (dctx->frameInfo.contentChecksumFlag)
-- 
cgit v0.12


From 2becd69bb1b29be8b1414c8ef6cc6269bb184f58 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 26 Apr 2018 17:25:12 -0400
Subject: Add _destSize() to Fullbench

---
 tests/fullbench.c | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/tests/fullbench.c b/tests/fullbench.c
index f489392..22199b7 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -184,6 +184,11 @@ static int local_LZ4_compress_default_small(const char* in, char* out, int inSiz
     return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1);
 }
 
+static int local_LZ4_compress_destSize(const char* in, char* out, int inSize)
+{
+    return LZ4_compress_destSize(in, out, &inSize, LZ4_compressBound(inSize)-1);
+}
+
 static int local_LZ4_compress_fast0(const char* in, char* out, int inSize)
 {
     return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 0);
@@ -422,12 +427,13 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
             case 0 : DISPLAY("Compression functions : \n"); continue;
             case 1 : compressionFunction = local_LZ4_compress_default_large; compressorName = "LZ4_compress_default"; break;
             case 2 : compressionFunction = local_LZ4_compress_default_small; compressorName = "LZ4_compress_default(small dst)"; break;
-            case 3 : compressionFunction = local_LZ4_compress_fast0; compressorName = "LZ4_compress_fast(0)"; break;
-            case 4 : compressionFunction = local_LZ4_compress_fast1; compressorName = "LZ4_compress_fast(1)"; break;
-            case 5 : compressionFunction = local_LZ4_compress_fast2; compressorName = "LZ4_compress_fast(2)"; break;
-            case 6 : compressionFunction = local_LZ4_compress_fast17; compressorName = "LZ4_compress_fast(17)"; break;
-            case 7 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break;
-            case 8 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break;
+            case 3 : compressionFunction = local_LZ4_compress_destSize; compressorName = "LZ4_compress_destSize"; break;
+            case 4 : compressionFunction = local_LZ4_compress_fast0; compressorName = "LZ4_compress_fast(0)"; break;
+            case 5 : compressionFunction = local_LZ4_compress_fast1; compressorName = "LZ4_compress_fast(1)"; break;
+            case 6 : compressionFunction = local_LZ4_compress_fast2; compressorName = "LZ4_compress_fast(2)"; break;
+            case 7 : compressionFunction = local_LZ4_compress_fast17; compressorName = "LZ4_compress_fast(17)"; break;
+            case 8 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break;
+            case 9 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break;
 
             case 10: compressionFunction = local_LZ4_compress_HC; compressorName = "LZ4_compress_HC"; break;
             case 12: compressionFunction = local_LZ4_compress_HC_extStateHC; compressorName = "LZ4_compress_HC_extStateHC"; break;
-- 
cgit v0.12


From 0858362f28195bb1d88bfa60075d3ec19ff73a9c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 26 Apr 2018 15:42:16 -0400
Subject: Merge _destSize Compress Variant into LZ4_compress_generic()

---
 lib/lz4.c | 256 ++++++++++++++++----------------------------------------------
 1 file changed, 66 insertions(+), 190 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index eaabe5b..b4d3140 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -453,7 +453,7 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 /*-************************************
 *  Local Structures and types
 **************************************/
-typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
+typedef enum { notLimited = 0, limitedOutput = 1, fillOutput = 2 } limitedOutput_directive;
 typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t;
 
 /**
@@ -637,6 +637,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                  const char* const source,
                  char* const dest,
                  const int inputSize,
+                 int *inputConsumed, /* only written when outputLimited == fillOutput */
                  const int maxOutputSize,
                  const limitedOutput_directive outputLimited,
                  const tableType_t tableType,
@@ -679,6 +680,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType);
     /* Init conditions */
+    if (outputLimited == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
     assert(acceleration >= 1);
@@ -795,9 +797,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         /* Encode Literals */
         {   unsigned const litLength = (unsigned)(ip - anchor);
             token = op++;
-            if ((outputLimited) &&  /* Check output buffer overflow */
+            if ((outputLimited == limitedOutput) &&  /* Check output buffer overflow */
                 (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
                 return 0;
+            if ((outputLimited == fillOutput) &&
+                (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) {
+                op--;
+                goto _last_literals;
+            }
             if (litLength >= RUN_MASK) {
                 int len = (int)litLength-RUN_MASK;
                 *token = (RUN_MASK<<ML_BITS);
@@ -822,6 +829,13 @@ _next_match:
          * - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written
          */
 
+        if ((outputLimited == fillOutput) &&
+            (op + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit)) {
+            /* the match was too close to the end, rewind and go to last literals */
+            op = token;
+            goto _last_literals;
+        }
+
         /* Encode Offset */
         if (maybe_extMem) {   /* static test */
             DEBUGLOG(6, "             with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
@@ -855,9 +869,17 @@ _next_match:
                 DEBUGLOG(6, "             with matchLength=%u", matchCode+MINMATCH);
             }
 
-            if ( outputLimited &&    /* Check output buffer overflow */
-                (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
-                return 0;
+            if ((outputLimited) &&    /* Check output buffer overflow */
+                (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) {
+                if (outputLimited == limitedOutput)
+                  return 0;
+                if (outputLimited == fillOutput) {
+                    /* Match description too long : reduce it */
+                    U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255;
+                    ip -= matchCode - newMatchCode;
+                    matchCode = newMatchCode;
+                }
+            }
             if (matchCode >= ML_MASK) {
                 *token += ML_MASK;
                 matchCode -= ML_MASK;
@@ -939,10 +961,17 @@ _next_match:
 
 _last_literals:
     /* Encode Last Literals */
-    {   size_t const lastRun = (size_t)(iend - anchor);
+    {   size_t lastRun = (size_t)(iend - anchor);
         if ( (outputLimited) &&  /* Check output buffer overflow */
-            ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) )
-            return 0;
+            (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) {
+            if (outputLimited == fillOutput) {
+                /* adapt lastRun to fill 'dst' */
+                lastRun  = (olimit-op) - 1;
+                lastRun -= (lastRun+240)/255;
+            }
+            if (outputLimited == limitedOutput)
+                return 0;
+        }
         if (lastRun >= RUN_MASK) {
             size_t accumulator = lastRun - RUN_MASK;
             *op++ = RUN_MASK << ML_BITS;
@@ -952,9 +981,13 @@ _last_literals:
             *op++ = (BYTE)(lastRun<<ML_BITS);
         }
         memcpy(op, anchor, lastRun);
+        ip = anchor + lastRun;
         op += lastRun;
     }
 
+    if (outputLimited == fillOutput) {
+        *inputConsumed = (int) (((const char*)ip)-source);
+    }
     return (int)(((char*)op) - dest);
 }
 
@@ -966,17 +999,17 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
     LZ4_resetStream((LZ4_stream_t*)state);
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {;
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
 }
@@ -1000,28 +1033,28 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
             const tableType_t tableType = byU16;
             LZ4_prepareTable(ctx, srcSize, tableType);
             if (ctx->currentOffset) {
-                return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
-                return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
             const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
             LZ4_prepareTable(ctx, srcSize, tableType);
-            return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (srcSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
             LZ4_prepareTable(ctx, srcSize, tableType);
             if (ctx->currentOffset) {
-                return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
-                return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
             const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
             LZ4_prepareTable(ctx, srcSize, tableType);
-            return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
 }
@@ -1060,172 +1093,15 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m
     LZ4_resetStream(&ctx);
 
     if (inputSize < LZ4_64Klimit)
-        return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16,                        noDict, noDictIssue, acceleration);
+        return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16,                        noDict, noDictIssue, acceleration);
     else
-        return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration);
-}
-
-
-/*-******************************
-*  *_destSize() variant
-********************************/
-
-static int LZ4_compress_destSize_generic(
-                       LZ4_stream_t_internal* const ctx,
-                 const char* const src,
-                       char* const dst,
-                       int*  const srcSizePtr,
-                 const int targetDstSize,
-                 const tableType_t tableType)
-{
-    const BYTE* ip = (const BYTE*) src;
-    const BYTE* base = (const BYTE*) src;
-    const BYTE* lowLimit = (const BYTE*) src;
-    const BYTE* anchor = ip;
-    const BYTE* const iend = ip + *srcSizePtr;
-    const BYTE* const mflimit = iend - MFLIMIT;
-    const BYTE* const matchlimit = iend - LASTLITERALS;
-
-    BYTE* op = (BYTE*) dst;
-    BYTE* const oend = op + targetDstSize;
-    BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */;
-    BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */);
-    BYTE* const oMaxSeq = oMaxLit - 1 /* token */;
-
-    U32 forwardH;
-
-
-    /* Init conditions */
-    if (targetDstSize < 1) return 0;                                     /* Impossible to store anything */
-    if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0;            /* Unsupported input size, too large (or negative) */
-    if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
-    if (*srcSizePtr<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
-
-    /* First Byte */
-    *srcSizePtr = 0;
-    LZ4_putPosition(ip, ctx->hashTable, tableType, base);
-    ip++; forwardH = LZ4_hashPosition(ip, tableType);
-
-    /* Main Loop */
-    for ( ; ; ) {
-        const BYTE* match;
-        BYTE* token;
-
-        /* Find a match */
-        {   const BYTE* forwardIp = ip;
-            unsigned step = 1;
-            unsigned searchMatchNb = 1 << LZ4_skipTrigger;
-
-            do {
-                U32 h = forwardH;
-                ip = forwardIp;
-                forwardIp += step;
-                step = (searchMatchNb++ >> LZ4_skipTrigger);
-
-                if (unlikely(forwardIp > mflimit)) goto _last_literals;
-
-                match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base);
-                forwardH = LZ4_hashPosition(forwardIp, tableType);
-                LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base);
-
-            } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
-                || (LZ4_read32(match) != LZ4_read32(ip)) );
-        }
-
-        /* Catch up */
-        while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
-
-        /* Encode Literal length */
-        {   unsigned litLength = (unsigned)(ip - anchor);
-            token = op++;
-            if (op + ((litLength+240)/255) + litLength > oMaxLit) {
-                /* Not enough space for a last match */
-                op--;
-                goto _last_literals;
-            }
-            if (litLength>=RUN_MASK) {
-                unsigned len = litLength - RUN_MASK;
-                *token=(RUN_MASK<<ML_BITS);
-                for(; len >= 255 ; len-=255) *op++ = 255;
-                *op++ = (BYTE)len;
-            }
-            else *token = (BYTE)(litLength<<ML_BITS);
-
-            /* Copy Literals */
-            LZ4_wildCopy(op, anchor, op+litLength);
-            op += litLength;
-        }
-
-_next_match:
-        /* Encode Offset */
-        LZ4_writeLE16(op, (U16)(ip-match)); op+=2;
-
-        /* Encode MatchLength */
-        {   size_t matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
-
-            if (op + ((matchLength+240)/255) > oMaxMatch) {
-                /* Match description too long : reduce it */
-                matchLength = (15-1) + (oMaxMatch-op) * 255;
-            }
-            ip += MINMATCH + matchLength;
-
-            if (matchLength>=ML_MASK) {
-                *token += ML_MASK;
-                matchLength -= ML_MASK;
-                while (matchLength >= 255) { matchLength-=255; *op++ = 255; }
-                *op++ = (BYTE)matchLength;
-            }
-            else *token += (BYTE)(matchLength);
-        }
-
-        anchor = ip;
-
-        /* Test end of block */
-        if (ip > mflimit) break;
-        if (op > oMaxSeq) break;
-
-        /* Fill table */
-        LZ4_putPosition(ip-2, ctx->hashTable, tableType, base);
-
-        /* Test next position */
-        match = LZ4_getPosition(ip, ctx->hashTable, tableType, base);
-        LZ4_putPosition(ip, ctx->hashTable, tableType, base);
-        if ( (match+MAX_DISTANCE>=ip)
-            && (LZ4_read32(match)==LZ4_read32(ip)) )
-        { token=op++; *token=0; goto _next_match; }
-
-        /* Prepare next loop */
-        forwardH = LZ4_hashPosition(++ip, tableType);
-    }
-
-_last_literals:
-    /* Encode Last Literals */
-    {   size_t lastRunSize = (size_t)(iend - anchor);
-        if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) {
-            /* adapt lastRunSize to fill 'dst' */
-            lastRunSize  = (oend-op) - 1;
-            lastRunSize -= (lastRunSize+240)/255;
-        }
-        ip = anchor + lastRunSize;
-
-        if (lastRunSize >= RUN_MASK) {
-            size_t accumulator = lastRunSize - RUN_MASK;
-            *op++ = RUN_MASK << ML_BITS;
-            for(; accumulator >= 255 ; accumulator-=255) *op++ = 255;
-            *op++ = (BYTE) accumulator;
-        } else {
-            *op++ = (BYTE)(lastRunSize<<ML_BITS);
-        }
-        memcpy(op, anchor, lastRunSize);
-        op += lastRunSize;
-    }
-
-    /* End */
-    *srcSizePtr = (int) (((const char*)ip)-src);
-    return (int) (((char*)op)-dst);
+        return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration);
 }
 
 
+/* Note!: This function leaves the stream in an unclean/broken state!
+ * It is not safe to subsequently use the same state with a _fastReset() or
+ * _continue() call without resetting it. */
 static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize)
 {
     LZ4_resetStream(state);
@@ -1234,10 +1110,10 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
         return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
     } else {
         if (*srcSizePtr < LZ4_64Klimit) {
-            return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16);
+            return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1);
         } else {
             tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
-            return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, tableType);
+            return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, tableType, noDict, noDictIssue, 1);
     }   }
 }
 
@@ -1397,9 +1273,9 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
-            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
-            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration);
     }
 
     /* external dictionary mode */
@@ -1417,15 +1293,15 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                  * so that the compression loop is only looking into one table.
                  */
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
         } else {
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             } else {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             }
         }
         streamPtr->dictionary = (const BYTE*)source;
@@ -1444,9 +1320,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     LZ4_renormDictT(streamPtr, srcSize);
 
     if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
     } else {
-        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
     }
 
     streamPtr->dictionary = (const BYTE*)source;
-- 
cgit v0.12


From 3792d00168edd060c58ceaecffb97d43dab27094 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 26 Apr 2018 15:18:44 -0700
Subject: favorDecSpeed feature can be triggered from lz4frame

and lz4hc.
---
 lib/lz4frame.c |  3 +++
 lib/lz4frame.h |  9 +++++----
 lib/lz4hc.c    |  8 ++++++--
 lib/lz4hc.h    | 18 +++++++++++++-----
 4 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index b616463..06a0f7b 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -612,6 +612,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         /* frame init only for blockLinked : blockIndependent will be init at each block */
         LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
     }
+    if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
+          LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, preferencesPtr->favorDecSpeed);
+    }
 
     /* Magic Number */
     LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index bd715bd..fb434ff 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -173,13 +173,14 @@ typedef struct {
 
 /*! LZ4F_preferences_t :
  *  makes it possible to supply detailed compression parameters to the stream interface.
- *  It's not required to set all fields, as long as the structure was initially memset() to zero.
+ *  Structure is presumed initially memset() to zero, representing default settings.
  *  All reserved fields must be set to zero. */
 typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
-  unsigned autoFlush;              /* 1 == always flush, to reduce usage of internal buffers */
-  unsigned reserved[4];            /* must be zero for forward compatibility */
+  int      compressionLevel;    /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
+  unsigned autoFlush;           /* 1: always flush, to reduce usage of internal buffers */
+  unsigned favorDecSpeed;       /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4LZ4HC_CLEVEL_OPT_MIN) */  /* >= v1.8.2 */
+  unsigned reserved[3];         /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
 
 LZ4FLIB_API int LZ4F_compressionLevel_max(void);
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 3593da7..b90d60b 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -876,6 +876,11 @@ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev
     LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
 }
 
+void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor)
+{
+    LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor!=0);
+}
+
 int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
@@ -1120,7 +1125,7 @@ static int LZ4HC_compress_optimal (
     const limitedOutput_directive limit,
     int const fullUpdate,
     const dictCtx_directive dict,
-    HCfavor_e favorDecSpeed
+    const HCfavor_e favorDecSpeed
     )
 {
 #define TRAILING_LITERALS 3
@@ -1136,7 +1141,6 @@ static int LZ4HC_compress_optimal (
     BYTE* oend = op + dstCapacity;
 
     /* init */
-    favorDecSpeed = favorCompressionRatio;
     DEBUGLOG(5, "LZ4HC_compress_optimal");
     *srcSizePtr = 0;
     if (limit == limitedDestSize) oend -= LASTLITERALS;   /* Hack for support LZ4 format restriction */
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 28e2528..bb5e073 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -152,7 +152,8 @@ struct LZ4HC_CCtx_internal
     uint32_t   dictLimit;       /* below that point, need extDict */
     uint32_t   lowLimit;        /* below that point, no more dict */
     uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
-    int        compressionLevel;
+    short      compressionLevel;
+    short      favorDecSpeed;
     const LZ4HC_CCtx_internal* dictCtx;
 };
 
@@ -169,7 +170,8 @@ struct LZ4HC_CCtx_internal
     unsigned int   dictLimit;        /* below that point, need extDict */
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
-    int            compressionLevel;
+    short          compressionLevel;
+    short          favorDecSpeed;
     const LZ4HC_CCtx_internal* dictCtx;
 };
 
@@ -253,9 +255,9 @@ LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") LZ4LIB_API  int   LZ4_resetStr
  * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`
  */
 int LZ4_compress_HC_destSize(void* LZ4HC_Data,
-                            const char* src, char* dst,
-                            int* srcSizePtr, int targetDstSize,
-                            int compressionLevel);
+                             const char* src, char* dst,
+                             int* srcSizePtr, int targetDstSize,
+                             int compressionLevel);
 
 /*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental)
  *  Similar as LZ4_compress_HC_continue(),
@@ -275,6 +277,12 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
  */
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+/*! LZ4_favorDecompressionSpeed() : v1.8.2 (experimental)
+ *  Parser will select decisions favoring decompression over compression ratio.
+ *  Only work at highest compression settings (level >= LZ4HC_CLEVEL_OPT_MIN)
+ */
+void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor);
+
 /*! LZ4_resetStreamHC_fast() :
  *  When an LZ4_streamHC_t is known to be in a internally coherent state,
  *  it can often be prepared for a new compression with almost no work, only
-- 
cgit v0.12


From 5c7d3812d90aeaf072d14f6b5d935711da6f14c7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 26 Apr 2018 15:49:32 -0700
Subject: fasterDecSpeed can be triggered from cli with --favor-decSpeed

---
 lib/lz4frame.c    | 2 +-
 lib/lz4hc.c       | 2 +-
 programs/lz4cli.c | 2 ++
 programs/lz4io.c  | 8 ++++++++
 programs/lz4io.h  | 7 ++++++-
 5 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 06a0f7b..9d88644 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -613,7 +613,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
     }
     if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
-          LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, preferencesPtr->favorDecSpeed);
+          LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
     }
 
     /* Magic Number */
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b90d60b..39ab5fb 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -718,7 +718,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
                             cParam.nbSearches, cParam.targetLength, limit,
                             cLevel == LZ4HC_CLEVEL_MAX,   /* ultra mode */
                             dict,
-                            favorDecompressionSpeed);
+                            ctx->favorDecSpeed);
     }
 }
 
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 42392eb..ba519b4 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -140,6 +140,7 @@ static int usage_advanced(const char* exeName)
     DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n");
     DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
     DISPLAY( "--[no-]sparse  : sparse mode (default:enabled on file, disabled on stdout)\n");
+    DISPLAY( "--favor-decSpeed: compressed files decompress faster, but are less compressed \n");
     DISPLAY( "Benchmark arguments : \n");
     DISPLAY( " -b#    : benchmark file(s), using # compression level (default : 1) \n");
     DISPLAY( " -e#    : test all compression levels from -bX to # (default : 1)\n");
@@ -355,6 +356,7 @@ int main(int argc, const char** argv)
                 if (!strcmp(argument,  "--no-content-size")) { LZ4IO_setContentSize(0); continue; }
                 if (!strcmp(argument,  "--sparse")) { LZ4IO_setSparseFile(2); continue; }
                 if (!strcmp(argument,  "--no-sparse")) { LZ4IO_setSparseFile(0); continue; }
+                if (!strcmp(argument,  "--favor-decSpeed")) { LZ4IO_favorDecSpeed(1); continue; }
                 if (!strcmp(argument,  "--verbose")) { displayLevel++; continue; }
                 if (!strcmp(argument,  "--quiet")) { if (displayLevel) displayLevel--; continue; }
                 if (!strcmp(argument,  "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
diff --git a/programs/lz4io.c b/programs/lz4io.c
index ccf4fa1..b52c1f3 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -116,6 +116,7 @@ static int g_blockIndependence = 1;
 static int g_sparseFileSupport = 1;
 static int g_contentSizeFlag = 0;
 static int g_useDictionary = 0;
+static unsigned g_favorDecSpeed = 0;
 static const char* g_dictionaryFilename = NULL;
 
 
@@ -221,6 +222,12 @@ int LZ4IO_setContentSize(int enable)
     return g_contentSizeFlag;
 }
 
+/* Default setting : 0 (disabled) */
+void LZ4IO_favorDecSpeed(int favor)
+{
+    g_favorDecSpeed = (favor!=0);
+}
+
 static U32 g_removeSrcFile = 0;
 void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
 
@@ -548,6 +555,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
     prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId;
     prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum;
     prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum;
+    prefs.favorDecSpeed = g_favorDecSpeed;
     if (g_contentSizeFlag) {
       U64 const fileSize = UTIL_getFileSize(srcFileName);
       prefs.frameInfo.contentSize = fileSize;   /* == 0 if input == stdin */
diff --git a/programs/lz4io.h b/programs/lz4io.h
index b21b8b6..22c5e3e 100644
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -94,10 +94,15 @@ int LZ4IO_setNotificationLevel(int level);
 /* Default setting : 0 (disabled) */
 int LZ4IO_setSparseFile(int enable);
 
-/* Default setting : 0 (disabled) */
+/* Default setting : 0 == no content size present in frame header */
 int LZ4IO_setContentSize(int enable);
 
+/* Default setting : 0 == src file preserved */
 void LZ4IO_setRemoveSrcFile(unsigned flag);
 
+/* Default setting : 0 == favor compression ratio
+ * Note : 1 only works for high compression levels (10+) */
+void LZ4IO_favorDecSpeed(int favor);
+
 
 #endif  /* LZ4IO_H_237902873 */
-- 
cgit v0.12


From 0fb3a3b199ed0a92a402c5979c514a329b85462a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 26 Apr 2018 17:02:20 -0700
Subject: fixed a number of minor cast warnings

---
 examples/frameCompress.c | 3 ++-
 lib/lz4frame.c           | 2 +-
 lib/lz4hc.c              | 9 ++++-----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 972f716..9bfea48 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -21,7 +21,8 @@ static const LZ4F_preferences_t kPrefs = {
       0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
     0,   /* compression level; 0 == default */
     0,   /* autoflush */
-    { 0, 0, 0, 0 },  /* reserved, must be set to 0 */
+    0,   /* favor decompression speed */
+    { 0, 0, 0 },  /* reserved, must be set to 0 */
 };
 
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 9d88644..0338266 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -613,7 +613,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
     }
     if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
-          LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
+          LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
     }
 
     /* Magic Number */
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 39ab5fb..35eac1a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -708,6 +708,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
     assert(cLevel >= 0);
     assert(cLevel <= LZ4HC_CLEVEL_MAX);
     {   cParams_t const cParam = clTable[cLevel];
+        HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio;
         if (cParam.strat == lz4hc)
             return LZ4HC_compress_hashChain(ctx,
                                 src, dst, srcSizePtr, dstCapacity,
@@ -717,8 +718,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
                             src, dst, srcSizePtr, dstCapacity,
                             cParam.nbSearches, cParam.targetLength, limit,
                             cLevel == LZ4HC_CLEVEL_MAX,   /* ultra mode */
-                            dict,
-                            ctx->favorDecSpeed);
+                            dict, favor);
     }
 }
 
@@ -756,7 +756,7 @@ static int LZ4HC_compress_generic_dictCtx (
     } else if (position == 0 && *srcSizePtr > 4 KB) {
         memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
         LZ4HC_setExternalDict(ctx, (const BYTE *)src);
-        ctx->compressionLevel = cLevel;
+        ctx->compressionLevel = (short)cLevel;
         return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
     } else {
         return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
@@ -873,7 +873,7 @@ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev
 {
     if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
     if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX;
-    LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
+    LZ4_streamHCPtr->internal_donotuse.compressionLevel = (short)compressionLevel;
 }
 
 void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor)
@@ -1274,7 +1274,6 @@ static int LZ4HC_compress_optimal (
                          price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
                      }
 
-                    assert(opt[pos].price > 1);
                     assert((U32)favorDecSpeed <= 1);
                      if (pos > last_match_pos+TRAILING_LITERALS
                       || price <= opt[pos].price - (int)favorDecSpeed) {
-- 
cgit v0.12


From ce4e1389cc38c3c26f40ecef9c4fa6438411fa90 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Fri, 27 Apr 2018 07:06:37 +0300
Subject: fuzzer.c: enabled ring buffer tests for decompress_fast

Ring buffer tests were performed only with LZ4_decompress_safe_continue,
leaving my buggy changes to LZ4_decompress_safe_continue undetected.
The tests are now replicated and performed in a similar manner for both
LZ4_decompress_safe_continue and LZ4_decompress_safe_continue (except
for the small buffer case where only one function can be tested,
because part of the dictionary is overwritten with the output).

I also updated function names in the messages (changed them to the
actual ones).  The error was reported for LZ4_decompress_safe(),
which I found misleading.
---
 tests/fuzzer.c | 121 ++++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 81 insertions(+), 40 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 244cc4f..cadda21 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -914,8 +914,8 @@ static void FUZ_unitTests(int compressionLevel)
 
         /* ring buffer test */
         {   XXH64_state_t xxhOrig;
-            XXH64_state_t xxhNew;
-            LZ4_streamDecode_t decodeState;
+            XXH64_state_t xxhNewSafe, xxhNewFast;
+            LZ4_streamDecode_t decodeStateSafe, decodeStateFast;
             const U32 maxMessageSizeLog = 10;
             const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
             U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
@@ -923,26 +923,36 @@ static void FUZ_unitTests(int compressionLevel)
             U32 rNext = 0;
             U32 dNext = 0;
             const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
+            int compressedSize;
 
             XXH64_reset(&xxhOrig, 0);
-            XXH64_reset(&xxhNew, 0);
+            XXH64_reset(&xxhNewSafe, 0);
+            XXH64_reset(&xxhNewFast, 0);
             LZ4_resetStream(&streamingState);
-            LZ4_setStreamDecode(&decodeState, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateSafe, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
 
             while (iNext + messageSize < testCompressedSize) {
                 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
                 crcOrig = XXH64_digest(&xxhOrig);
 
                 memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
-                result = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1);
-                FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed");
+                compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1);
+                FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_fast_continue() compression failed");
 
-                result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
-                FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
+                result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
+                FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed");
 
-                XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-                { U64 const crcNew = XXH64_digest(&xxhNew);
-                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
+                XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+                { U64 const crcNew = XXH64_digest(&xxhNewSafe);
+                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
+
+                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+                FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed");
+
+                XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+                { U64 const crcNew = XXH64_digest(&xxhNewFast);
+                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
 
                 /* prepare next message */
                 iNext += messageSize;
@@ -1064,8 +1074,8 @@ static void FUZ_unitTests(int compressionLevel)
 
         /* ring buffer test */
         {   XXH64_state_t xxhOrig;
-            XXH64_state_t xxhNew;
-            LZ4_streamDecode_t decodeState;
+            XXH64_state_t xxhNewSafe, xxhNewFast;
+            LZ4_streamDecode_t decodeStateSafe, decodeStateFast;
             const U32 maxMessageSizeLog = 10;
             const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
             U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
@@ -1073,26 +1083,36 @@ static void FUZ_unitTests(int compressionLevel)
             U32 rNext = 0;
             U32 dNext = 0;
             const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
+            int compressedSize;
 
             XXH64_reset(&xxhOrig, 0);
-            XXH64_reset(&xxhNew, 0);
+            XXH64_reset(&xxhNewSafe, 0);
+            XXH64_reset(&xxhNewFast, 0);
             LZ4_resetStreamHC(&sHC, compressionLevel);
-            LZ4_setStreamDecode(&decodeState, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateSafe, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
 
             while (iNext + messageSize < testCompressedSize) {
                 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
                 crcOrig = XXH64_digest(&xxhOrig);
 
                 memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
-                result = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
-                FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
+                compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+                FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
+
+                result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
+                FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed");
 
-                result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
-                FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
+                XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+                { U64 const crcNew = XXH64_digest(&xxhNewSafe);
+                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
 
-                XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-                { U64 const crcNew = XXH64_digest(&xxhNew);
-                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
+                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+                FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed");
+
+                XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+                { U64 const crcNew = XXH64_digest(&xxhNewFast);
+                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
 
                 /* prepare next message */
                 iNext += messageSize;
@@ -1106,8 +1126,8 @@ static void FUZ_unitTests(int compressionLevel)
 
         /* small decoder-side ring buffer test */
         {   XXH64_state_t xxhOrig;
-            XXH64_state_t xxhNew;
-            LZ4_streamDecode_t decodeState;
+            XXH64_state_t xxhNewSafe, xxhNewFast;
+            LZ4_streamDecode_t decodeStateSafe, decodeStateFast;
             const U32 maxMessageSizeLog = 12;
             const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
             U32 messageSize;
@@ -1115,11 +1135,14 @@ static void FUZ_unitTests(int compressionLevel)
             U32 iNext = 0;
             U32 dNext = 0;
             const U32 dBufferSize = 64 KB;
+            int compressedSize;
 
             XXH64_reset(&xxhOrig, 0);
-            XXH64_reset(&xxhNew, 0);
+            XXH64_reset(&xxhNewSafe, 0);
+            XXH64_reset(&xxhNewFast, 0);
             LZ4_resetStreamHC(&sHC, compressionLevel);
-            LZ4_setStreamDecode(&decodeState, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateSafe, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
 
 #define BSIZE1 65537
 #define BSIZE2 16435
@@ -1129,15 +1152,22 @@ static void FUZ_unitTests(int compressionLevel)
             XXH64_update(&xxhOrig, testInput + iNext, messageSize);
             crcOrig = XXH64_digest(&xxhOrig);
 
-            result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
-            FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
+            compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+            FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
+
+            result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
+            FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
 
-            result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
-            FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
+            XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+            { U64 const crcNew = XXH64_digest(&xxhNewSafe);
+              FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
 
-            XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-            { U64 const crcNew = XXH64_digest(&xxhNew);
-              FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
+            result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+            FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed");
+
+            XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+            { U64 const crcNew = XXH64_digest(&xxhNewFast);
+              FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
 
             /* prepare next message */
             dNext += messageSize;
@@ -1151,17 +1181,28 @@ static void FUZ_unitTests(int compressionLevel)
                 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
                 crcOrig = XXH64_digest(&xxhOrig);
 
-                result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
-                FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
+                compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+                FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
 
-                result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
-                FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
+#if 1           /* Because the ring buffer is small, decompression overwrites part of the output which
+                 * is first used as dictionary.  Hence only one decompression function can be tested. */
+                result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
+                FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
+                XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+                {   U64 const crcNew = XXH64_digest(&xxhNewSafe);
+                    if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, testVerify + dNext);
+                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during small decoder-side ring buffer test");
+                }
+#else
+                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+                FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed");
 
-                XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-                {   U64 const crcNew = XXH64_digest(&xxhNew);
+                XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+                {   U64 const crcNew = XXH64_digest(&xxhNewFast);
                     if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, testVerify + dNext);
-                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test");
+                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during small decoder-side ring buffer test");
                 }
+#endif
                 /* prepare next message */
                 dNext += messageSize;
                 totalMessageSize += messageSize;
-- 
cgit v0.12


From d81a434c3dc032779dba6f22a58b127922ef332f Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Fri, 27 Apr 2018 15:00:11 +0300
Subject: lz4.c: fixed the LZ4_decompress_fast_continue case

The change is very similar to that of the LZ4_decompress_safe_continue
case.  The only reason a make this a separate change is to ensure that
the fuzzer, after it's been enhanced, can detect the flaw in
LZ4_decompress_fast_continue, and that the change indeed fixes the flaw.
---
 lib/lz4.c | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index eb3da21..916acf0 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1752,6 +1752,15 @@ int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compresse
                                   (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
 }
 
+LZ4_FORCE_INLINE
+int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize,
+                                   size_t prefixSize, const void* dictStart, size_t dictSize)
+{
+    return LZ4_decompress_generic(source, dest, 0, originalSize,
+                                  endOnOutputSize, full, 0, usingExtDict,
+                                  (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
+}
+
 /*===== streaming decompression functions =====*/
 
 LZ4_streamDecode_t* LZ4_createStreamDecode(void)
@@ -1831,21 +1840,32 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
     return result;
 }
 
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)
 {
     LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
     int result;
 
-    if (lz4sd->prefixSize == 0 || lz4sd->prefixEnd == (BYTE*)dest) {
+    if (lz4sd->prefixSize == 0) {
+        assert(lz4sd->extDictSize == 0);
         result = LZ4_decompress_fast(source, dest, originalSize);
         if (result <= 0) return result;
+        lz4sd->prefixSize = originalSize;
+        lz4sd->prefixEnd = (BYTE*)dest + originalSize;
+    } else if (lz4sd->prefixEnd == (BYTE*)dest) {
+        if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0)
+            result = LZ4_decompress_fast(source, dest, originalSize);
+        else
+            result = LZ4_decompress_fast_doubleDict(source, dest, originalSize,
+                                                    lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
+        if (result <= 0) return result;
         lz4sd->prefixSize += originalSize;
         lz4sd->prefixEnd  += originalSize;
     } else {
         lz4sd->extDictSize = lz4sd->prefixSize;
         lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
         result = LZ4_decompress_fast_extDict(source, dest, originalSize,
-                                             (const char*)lz4sd->externalDict, lz4sd->extDictSize);
+                                             lz4sd->externalDict, lz4sd->extDictSize);
         if (result <= 0) return result;
         lz4sd->prefixSize = originalSize;
         lz4sd->prefixEnd  = (BYTE*)dest + originalSize;
-- 
cgit v0.12


From 938e4849ae3bb33665aeee2647aa3e2077af0b5a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 27 Apr 2018 08:43:40 -0700
Subject: updated NEWS, in preparation for v1.8.2

---
 NEWS | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/NEWS b/NEWS
index 9ac41f2..c3b47f1 100644
--- a/NEWS
+++ b/NEWS
@@ -1,7 +1,13 @@
 v1.8.2
+perf: *much* faster dictionary compression on small files, by @felixhandte
 perf: slightly faster HC compression and decompression speed
 perf: very small compression ratio improvement
+fix : compression compatible with low memory addresses (< 0xFFFF)
+fix : decompression segfault when provided with NULL input, by @terrelln
+cli : new command --favor-decSpeed
 cli : benchmark mode more accurate for small inputs
+fullbench : can measure _destSize() variants, by @felixhandte
+doc : clarified block format parsing restrictions, by Alexey Tourbin (@svpv)
 
 v1.8.1
 perf : faster and stronger ultra modes (levels 10+)
-- 
cgit v0.12


From d294dd7fc684b1f46130358ff5c218e835082b16 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 27 Apr 2018 09:04:09 -0700
Subject: ensure favorDecSpeed is properly initialized

also :
- fix a potential malloc error
- proper use of ALLOC macro inside lz4hc
- update html API doc
---
 doc/lz4_manual.html      | 12 +++++----
 doc/lz4frame_manual.html | 63 +++++++++++++++++++++++++++++++++++++++++++++---
 lib/lz4.c                |  6 ++---
 lib/lz4hc.c              | 10 ++++----
 4 files changed, 74 insertions(+), 17 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index f8639fe..ddd2724 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -126,15 +126,17 @@ int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int src
 
 <pre><b>int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
 </b><p>This function is a bit faster than LZ4_decompress_safe(),
-but doesn't provide any security guarantee.
+but it may misbehave on malformed input because it doesn't perform full validation of compressed data.
     originalSize : is the uncompressed size to regenerate
                    Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
     return : number of bytes read from source buffer (== compressed size).
              If the source stream is detected malformed, the function stops decoding and return a negative result.
-    note : This function respects memory boundaries for *properly formed* compressed data.
-           However, it does not provide any protection against malicious input.
-           It also doesn't know 'src' size, and implies it's >= compressed size.
-           Use this function in trusted environment **only**.
+    note : This function is only usable if the originalSize of uncompressed data is known in advance.
+           The caller should also check that all the compressed input has been consumed properly,
+           i.e. that the return value matches the size of the buffer with compressed input.
+           The function never writes past the output buffer.  However, since it doesn't know its 'src' size,
+           it may read past the intended input.  Also, because match offsets are not validated during decoding,
+           reads from 'src' may underflow.  Use this function in trusted environment **only**.
 </p></pre><BR>
 
 <pre><b>int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 459bac8..53ea7eb 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -18,6 +18,7 @@
 <li><a href="#Chapter8">Compression</a></li>
 <li><a href="#Chapter9">Decompression functions</a></li>
 <li><a href="#Chapter10">Streaming decompression functions</a></li>
+<li><a href="#Chapter11">Bulk processing dictionary API</a></li>
 </ol>
 <hr>
 <a name="Chapter1"></a><h2>Introduction</h2><pre>
@@ -89,12 +90,13 @@
 
 <pre><b>typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       </b>/* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */<b>
-  unsigned autoFlush;              </b>/* 1 == always flush, to reduce usage of internal buffers */<b>
-  unsigned reserved[4];            </b>/* must be zero for forward compatibility */<b>
+  int      compressionLevel;    </b>/* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */<b>
+  unsigned autoFlush;           </b>/* 1: always flush, to reduce usage of internal buffers */<b>
+  unsigned favorDecSpeed;       </b>/* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4LZ4HC_CLEVEL_OPT_MIN) */  /* >= v1.8.2 */<b>
+  unsigned reserved[3];         </b>/* must be zero for forward compatibility */<b>
 } LZ4F_preferences_t;
 </b><p>  makes it possible to supply detailed compression parameters to the stream interface.
-  It's not required to set all fields, as long as the structure was initially memset() to zero.
+  Structure is presumed initially memset() to zero, representing default settings.
   All reserved fields must be set to zero. 
 </p></pre><BR>
 
@@ -293,5 +295,58 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
   and start a new one using same context resources. 
 </p></pre><BR>
 
+<pre><b>typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
+</b></pre><BR>
+<a name="Chapter11"></a><h2>Bulk processing dictionary API</h2><pre></pre>
+
+<pre><b>LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
+LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
+</b><p>  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict 
+</p></pre><BR>
+
+<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
+    LZ4F_cctx* cctx,
+    void* dst, size_t dstCapacity,
+    const void* src, size_t srcSize,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* preferencesPtr);
+</b><p>  Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+  cctx must point to a context created by LZ4F_createCompressionContext().
+  If cdict==NULL, compress without a dictionary.
+  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+  If this condition is not respected, function will fail (@return an errorCode).
+  The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+  but it's not recommended, as it's the only way to provide dictID in the frame header.
+ @return : number of bytes written into dstBuffer.
+           or an error code if it fails (can be tested using LZ4F_isError()) 
+</p></pre><BR>
+
+<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
+    LZ4F_cctx* cctx,
+    void* dstBuffer, size_t dstCapacity,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* prefsPtr);
+</b><p>  Inits streaming dictionary compression, and writes the frame header into dstBuffer.
+  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ `prefsPtr` is optional : you may provide NULL as argument,
+  however, it's the only way to provide dictID in the frame header.
+ @return : number of bytes written into dstBuffer for the header,
+           or an error code (which can be tested using LZ4F_isError()) 
+</p></pre><BR>
+
+<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
+    LZ4F_dctx* dctxPtr,
+    void* dstBuffer, size_t* dstSizePtr,
+    const void* srcBuffer, size_t* srcSizePtr,
+    const void* dict, size_t dictSize,
+    const LZ4F_decompressOptions_t* decompressOptionsPtr);
+</b><p>  Same as LZ4F_decompress(), using a predefined dictionary.
+  Dictionary is used "in place", without any preprocessing.
+  It must remain accessible throughout the entire frame decoding. 
+</p></pre><BR>
+
 </html>
 </body>
diff --git a/lib/lz4.c b/lib/lz4.c
index eaabe5b..deaec7b 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -161,11 +161,11 @@
 *  Memory routines
 **************************************/
 #include <stdlib.h>   /* malloc, calloc, free */
-#define ALLOC(s) malloc(s)
+#define ALLOC(s)          malloc(s)
 #define ALLOC_AND_ZERO(s) calloc(1,s)
-#define FREEMEM        free
+#define FREEMEM(p)        free(p)
 #include <string.h>   /* memset, memcpy */
-#define MEM_INIT       memset
+#define MEM_INIT(p,v,s)   memset((p),(v),(s))
 
 
 /*-************************************
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 35eac1a..1c7096b 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -805,7 +805,7 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
 int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
 {
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
-    LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
+    LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
 #else
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
@@ -834,10 +834,9 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 **************************************/
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
-    LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
-    LZ4_streamHCPtr->internal_donotuse.base = NULL;
-    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
+    if (LZ4_streamHCPtr==NULL) return NULL;
+    LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT);
     return LZ4_streamHCPtr;
 }
 
@@ -857,6 +856,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = 0;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
 
-- 
cgit v0.12


From 7d11e344130a16a529e3ee15a16a10f2e67faa3c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 27 Apr 2018 13:57:10 -0400
Subject: Rename LZ4F_applyCDict() -> LZ4F_initStream()

---
 lib/lz4frame.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 4d6d39c..e2be990 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -528,7 +528,15 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 }
 
 
-static void LZ4F_applyCDict(void* ctx,
+/**
+ * This function prepares the internal LZ4(HC) stream for a new compression,
+ * resetting the context and attaching the dictionary, if there is one.
+ *
+ * It needs to be called at the beginning of each independent compression
+ * stream (i.e., at the beginning of a frame in blockLinked mode, or at the
+ * beginning of each block in blockIndependent mode).
+ */
+static void LZ4F_initStream(void* ctx,
                             const LZ4F_CDict* cdict,
                             int level) {
     if (level < LZ4HC_CLEVEL_MIN) {
@@ -610,7 +618,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->cdict = cdict;
     if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
         /* frame init only for blockLinked : blockIndependent will be init at each block */
-        LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
+        LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
     }
 
     /* Magic Number */
@@ -705,7 +713,7 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
-    LZ4F_applyCDict(ctx, cdict, level);
+    LZ4F_initStream(ctx, cdict, level);
     if (cdict) {
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
@@ -722,7 +730,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
 
 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
-    LZ4F_applyCDict(ctx, cdict, level);
+    LZ4F_initStream(ctx, cdict, level);
     if (cdict) {
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }
-- 
cgit v0.12


From 5076aa3e352ecbd67cd71314cb1fafcb319aed56 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 27 Apr 2018 13:59:02 -0400
Subject: Remove Redundant LZ4_resetStream() Call

---
 lib/lz4frame.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index e2be990..c4e0ae0 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -457,7 +457,7 @@ struct LZ4F_CDict_s {
 LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
     const char* dictStart = (const char*)dictBuffer;
-    LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
+    LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));
     if (!cdict) return NULL;
     if (dictSize > 64 KB) {
         dictStart += dictSize - 64 KB;
@@ -471,7 +471,6 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
         return NULL;
     }
     memcpy(cdict->dictContent, dictStart, dictSize);
-    LZ4_resetStream(cdict->fastCtx);
     LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
     LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
     LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);
-- 
cgit v0.12


From fefc40fc0afe8c1f5a02e65b94bb65515fd949c4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 27 Apr 2018 14:10:27 -0400
Subject: Avoid Possibly Redundant Table Clears When Loading HC Dict

---
 lib/lz4frame.c | 2 +-
 lib/lz4hc.c    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index c4e0ae0..8cf0c2f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -472,7 +472,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
     }
     memcpy(cdict->dictContent, dictStart, dictSize);
     LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
-    LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
+    LZ4_setCompressionLevel(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
     LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);
     return cdict;
 }
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 4126ef8..90f52f0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -877,8 +877,8 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
     }
+    LZ4_resetStreamHC(LZ4_streamHCPtr, ctxPtr->compressionLevel);
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
-    LZ4HC_clearTables (ctxPtr);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
     if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;
-- 
cgit v0.12


From 72e99c8939d7f005487403ab54fa65ac31ee4317 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 27 Apr 2018 11:44:47 -0700
Subject: lz4hc : minor editions for clarity

---
 lib/lz4hc.c | 75 ++++++++++++++++++++++++++++++-------------------------------
 1 file changed, 37 insertions(+), 38 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 4126ef8..0778552 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -138,6 +138,7 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
 {
     int back = 0;
     int const min = (int)MAX(iMin - ip, mMin - match);
+    assert(min <= 0);
     assert(ip >= iMin); assert((size_t)(ip-iMin) < (1U<<31));
     assert(match >= mMin); assert((size_t)(match - mMin) < (1U<<31));
     while ( (back > min)
@@ -222,7 +223,7 @@ LZ4HC_InsertAndGetWiderMatch (
     const U32 ipIndex = (U32)(ip - base);
     const U32 lowLimit = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
     const BYTE* const dictBase = hc4->dictBase;
-    int const delta = (int)(ip-iLowLimit);
+    int const lookBackLength = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
     U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
@@ -242,34 +243,35 @@ LZ4HC_InsertAndGetWiderMatch (
         nbAttempts--;
         if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
+            assert(matchPtr >= lowPrefixPtr);
+            assert(matchPtr < ip);
             assert(longest >= 1);
-            if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - delta + longest - 1)) {
+            if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
                     int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-                    int const back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
+                    int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
                     mlt -= back;
-
                     if (mlt > longest) {
                         longest = mlt;
                         *matchpos = matchPtr+back;
                         *startpos = ip+back;
-                }   }
-            }
+            }   }   }
         } else {   /* matchIndex < dictLimit */
             const BYTE* const matchPtr = dictBase + matchIndex;
             if (LZ4_read32(matchPtr) == pattern) {
+                const BYTE* const dictLowLimit = dictBase + hc4->lowLimit;
                 int mlt;
                 int back = 0;
                 const BYTE* vLimit = ip + (dictLimit - matchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
                 if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
-                    mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
-                back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictBase+lowLimit) : 0;
+                    mlt += LZ4_count(ip+mlt, lowPrefixPtr, iHighLimit);
+                back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictLowLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {
                     longest = mlt;
-                    *matchpos = base + matchIndex + back;
+                    *matchpos = base + matchIndex + back;   /* virtual pos, relative to ip, to retrieve offset */
                     *startpos = ip + back;
         }   }   }
 
@@ -317,7 +319,7 @@ LZ4HC_InsertAndGetWiderMatch (
                 const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
-                back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
+                back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {
                     longest = mlt;
@@ -456,14 +458,14 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
     BYTE* op = (BYTE*) dest;
     BYTE* oend = op + maxOutputSize;
 
-    int   ml, ml2, ml3, ml0;
+    int   ml0, ml, ml2, ml3;
+    const BYTE* start0;
+    const BYTE* ref0;
     const BYTE* ref = NULL;
     const BYTE* start2 = NULL;
     const BYTE* ref2 = NULL;
     const BYTE* start3 = NULL;
     const BYTE* ref3 = NULL;
-    const BYTE* start0;
-    const BYTE* ref0;
 
     /* init */
     *srcSizePtr = 0;
@@ -476,31 +478,27 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
         if (ml<MINMATCH) { ip++; continue; }
 
         /* saved, in case we would skip too much */
-        start0 = ip;
-        ref0 = ref;
-        ml0 = ml;
+        start0 = ip; ref0 = ref; ml0 = ml;
 
 _Search2:
-        if (ip+ml <= mflimit)
+        if (ip+ml <= mflimit) {
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
                             maxNbAttempts, patternAnalysis, dict);
-        else
+        } else {
             ml2 = ml;
+        }
 
-        if (ml2 == ml) { /* No better match */
+        if (ml2 == ml) { /* No better match => encode ML1 */
             optr = op;
             if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
             continue;
         }
 
-        if (start0 < ip) {
-            if (start2 < ip + ml0) {  /* empirical */
-                ip = start0;
-                ref = ref0;
-                ml = ml0;
-            }
-        }
+        if (start0 < ip) {   /* first match was skipped at least once */
+            if (start2 < ip + ml0) {  /* squeezing ML1 between ML0(original ML1) and ML2 */
+                ip = start0; ref = ref0; ml = ml0;  /* restore initial ML1 */
+        }   }
 
         /* Here, start0==ip */
         if ((start2 - ip) < 3) {  /* First Match too small : removed */
@@ -528,14 +526,15 @@ _Search3:
         }
         /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
 
-        if (start2 + ml2 <= mflimit)
+        if (start2 + ml2 <= mflimit) {
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
                             maxNbAttempts, patternAnalysis, dict);
-        else
+        } else {
             ml3 = ml2;
+        }
 
-        if (ml3 == ml2) {  /* No better match : 2 sequences to encode */
+        if (ml3 == ml2) {  /* No better match => encode ML1 and ML2 */
             /* ip & ref are known; Now for ml */
             if (start2 < ip+ml)  ml = (int)(start2 - ip);
             /* Now, encode 2 sequences */
@@ -580,11 +579,12 @@ _Search3:
         }
 
         /*
-        * OK, now we have 3 ascending matches; let's write at least the first one
-        * ip & ref are known; Now for ml
+        * OK, now we have 3 ascending matches;
+        * let's write the first one ML1.
+        * ip & ref are known; Now decide ml.
         */
         if (start2 < ip+ml) {
-            if ((start2 - ip) < (int)ML_MASK) {
+            if ((start2 - ip) < OPTIMAL_ML) {
                 int correction;
                 if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
                 if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;
@@ -601,14 +601,13 @@ _Search3:
         optr = op;
         if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
 
-        ip = start2;
-        ref = ref2;
-        ml = ml2;
+        /* ML2 becomes ML1 */
+        ip = start2; ref = ref2; ml = ml2;
 
-        start2 = start3;
-        ref2 = ref3;
-        ml2 = ml3;
+        /* ML3 becomes ML2 */
+        start2 = start3; ref2 = ref3; ml2 = ml3;
 
+        /* let's find a new ML3 */
         goto _Search3;
     }
 
-- 
cgit v0.12


From 19b1267d44a6bd3fbb1b682d515a3ab5dc70ec50 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 27 Apr 2018 12:46:49 -0700
Subject: fix lz4hc -BD non-determinism

related to chain table update
---
 lib/lz4hc.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 0778552..184ca8c 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -699,8 +699,6 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
     ctx->end += *srcSizePtr;
     if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT;   /* note : convention is different from lz4frame, maybe something to review */
     cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel);
-    assert(cLevel >= 0);
-    assert(cLevel <= LZ4HC_CLEVEL_MAX);
     {   cParams_t const cParam = clTable[cLevel];
         if (cParam.strat == lz4hc)
             return LZ4HC_compress_hashChain(ctx,
@@ -892,7 +890,8 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
     DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
-    if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
+    if (ctxPtr->end >= ctxPtr->base + ctxPtr->dictLimit + 4)
+        LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
     ctxPtr->lowLimit  = ctxPtr->dictLimit;
-- 
cgit v0.12


From 69242a8a0821fb3157f1d28e0c07f3210a7fca52 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Sat, 28 Apr 2018 07:16:46 +0300
Subject: lib/Makefile: show commands with V=1

`make V=1` will now show the commands executed to build the library.
A similar technique is used in e.g. linux/Makefile.

The bulk of this change is produced with the following vim command:

    :g!/^\t@echo\>/s/^\t@/\t\$(Q)/
---
 lib/Makefile | 62 +++++++++++++++++++++++++++++++++---------------------------
 1 file changed, 34 insertions(+), 28 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index bb45582..d63de18 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -85,32 +85,38 @@ all: lib
 all32: CFLAGS+=-m32
 all32: all
 
+ifeq ($(V), 1)
+Q =
+else
+Q = @
+endif
+
 liblz4.a: $(SRCFILES)
 ifeq ($(BUILD_STATIC),yes)  # can be disabled on command line
 	@echo compiling static library
-	@$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
-	@$(AR) rcs $@ *.o
+	$(Q)$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
+	$(Q)$(AR) rcs $@ *.o
 endif
 
 $(LIBLZ4): $(SRCFILES)
 ifeq ($(BUILD_SHARED),yes)  # can be disabled on command line
 	@echo compiling dynamic library $(LIBVER)
 ifneq (,$(filter Windows%,$(OS)))
-	@$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
+	$(Q)$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
 	dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib
 else
-	@$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
+	$(Q)$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
-	@ln -sf $@ liblz4.$(SHARED_EXT_MAJOR)
-	@ln -sf $@ liblz4.$(SHARED_EXT)
+	$(Q)ln -sf $@ liblz4.$(SHARED_EXT_MAJOR)
+	$(Q)ln -sf $@ liblz4.$(SHARED_EXT)
 endif
 endif
 
 liblz4: $(LIBLZ4)
 
 clean:
-	@$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib
-	@$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
+	$(Q)$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib
+	$(Q)$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
 	@echo Cleaning library completed
 
 
@@ -152,41 +158,41 @@ INSTALL_DATA    ?= $(INSTALL) -m 644
 
 liblz4.pc: liblz4.pc.in Makefile
 	@echo creating pkgconfig
-	@sed -e 's|@PREFIX@|$(PREFIX)|' \
+	$(Q)sed -e 's|@PREFIX@|$(PREFIX)|' \
          -e 's|@LIBDIR@|$(LIBDIR)|' \
          -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
          -e 's|@VERSION@|$(LIBVER)|' \
           $< >$@
 
 install: lib liblz4.pc
-	@$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ $(DESTDIR)$(LIBDIR)/
-	@$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/
+	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ $(DESTDIR)$(LIBDIR)/
+	$(Q)$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/
 	@echo Installing libraries
 ifeq ($(BUILD_STATIC),yes)
-	@$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
-	@$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
+	$(Q)$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
+	$(Q)$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
 endif
 ifeq ($(BUILD_SHARED),yes)
-	@$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
-	@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
-	@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
+	$(Q)$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
+	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
+	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
 endif
 	@echo Installing headers in $(INCLUDEDIR)
-	@$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
-	@$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
-	@$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
+	$(Q)$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
+	$(Q)$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
+	$(Q)$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
 	@echo lz4 libraries installed
 
 uninstall:
-	@$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc
-	@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
-	@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
-	@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
-	@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
 	@echo lz4 libraries successfully uninstalled
 
 endif
-- 
cgit v0.12


From 45f8603aae389d34c689d3ff7427b314071ccd2c Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Sat, 28 Apr 2018 11:14:40 +0300
Subject: lz4.c: two-stage shortcut for LZ4_decompress_generic

---
 lib/lz4.c | 107 +++++++++++++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 82 insertions(+), 25 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 06ff611..1330374 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1397,6 +1397,9 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
     const int safeDecode = (endOnInput==endOnInputSize);
     const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
 
+    /* Set up the "end" pointers for the shortcut. */
+    const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/;
+    const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/;
 
     /* Special cases */
     if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT;                      /* targetOutputSize too high => just decode everything */
@@ -1406,39 +1409,90 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 
     /* Main Loop : decode sequences */
     while (1) {
-        size_t length;
         const BYTE* match;
         size_t offset;
 
         unsigned const token = *ip++;
+        size_t length = token >> ML_BITS; /* literal length */
 
         assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
-        /* shortcut for common case :
-         * in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes).
-         * this shortcut was tested on x86 and x64, where it improves decoding speed.
-         * it has not yet been benchmarked on ARM, Power, mips, etc.
-         * NOTE: The loop begins with a read, so we must have one byte left at the end. */
-        if (endOnInput
-          && ((ip + 14 /*maxLL*/ + 2 /*offset*/ < iend)
-            & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend)
-            & (token < (15<<ML_BITS))
-            & ((token & ML_MASK) != 15) ) ) {
-            size_t const ll = token >> ML_BITS;
-            size_t const off = LZ4_readLE16(ip+ll);
-            const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */
-            if ((off >= 8) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
-                size_t const ml = (token & ML_MASK) + MINMATCH;
-                memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/;
-                memcpy(op + 0, matchPtr + 0, 8);
-                memcpy(op + 8, matchPtr + 8, 8);
-                memcpy(op +16, matchPtr +16, 2);
-                op += ml;
-                continue;
+
+        /* A two-stage shortcut for the most common case:
+         * 1) If the literal length is 0..14, and there is enough space,
+         * enter the shortcut and copy 16 bytes on behalf of the literals
+         * (in the fast mode, only 8 bytes can be safely copied this way).
+         * 2) Further if the match length is 4..18, copy 18 bytes in a similar
+         * manner; but we ensure that there's enough space in the output for
+         * those 18 bytes earlier, upon entering the shortcut (in other words,
+         * there is a combined check for both stages).
+         */
+        if ((endOnInput ? length != RUN_MASK : length <= 8) &&
+            /* strictly "less than" on input, to re-enter the loop with at least one byte */
+            likely((endOnInput ? ip < shortiend : 1) && (op <= shortoend)))
+        {
+            /* Can we copy the literals with a single memcpy invocation?  Sometimes we can't
+             * copy 16 bytes, because they can clobber the dictionary in the ring buffer. */
+            if (!endOnInput /* only 8 bytes */ || /* nothing to clobber */ dict != usingExtDict) {
+                /* Copy the literals. */
+                memcpy(op, ip, endOnInput ? 16 : 8);
+                op += length; ip += length;
+
+                /* The second stage: prepare for match copying, decode full info.
+                 * It if doesn't work out, the info won't be wasted. */
+                length = token & ML_MASK; /* match length */
+                offset = LZ4_readLE16(ip); ip += 2;
+                match = op - offset;
+
+                /* Do not deal with overlapping matches. */
+                if ((length != 15) && (offset >= 8) &&
+                    (dict==withPrefix64k || match >= lowPrefix))
+                {
+                    /* Copy the match. */
+                    memcpy(op + 0, match + 0, 8);
+                    memcpy(op + 8, match + 8, 8);
+                    memcpy(op +16, match +16, 2);
+                    op += length + MINMATCH;
+                    /* Both stages worked, load the next token. */
+                    continue;
+                }
+            } else {
+                /* Save the literal length, can't copy 16 bytes just yet. */
+                size_t ll = length;
+
+                /* Prepare for the second satge. */
+                length = token & ML_MASK;
+                offset = LZ4_readLE16(ip+ll);
+                match = op + ll - offset;
+
+                if ((length != 15) && (offset >= 8) &&
+                    (dict==withPrefix64k || match >= lowPrefix))
+                {
+                    /* Copy the literals. */
+                    memcpy(op, ip, 16);
+                    op += ll; ip += ll + 2;
+                    /* Copy the match. */
+                    memcpy(op + 0, match + 0, 8);
+                    memcpy(op + 8, match + 8, 8);
+                    memcpy(op +16, match +16, 2);
+                    op += length + MINMATCH;
+                    /* Both stages worked, load the next token. */
+                    continue;
+                }
+
+                /* So we took the literlas, but the second stage didn't work. */
+                memcpy(op, ip, 8);
+                if (ll > 8)
+                    memcpy(op + 8, ip + 8, 8);
+                op += ll; ip += ll + 2;
             }
+
+            /* The second stage didn't work out, but the info is ready.
+             * Propel it right to the point of match copying. */
+            goto _copy_match;
         }
 
         /* decode literal length */
-        if ((length=(token>>ML_BITS)) == RUN_MASK) {
+        if (length == RUN_MASK) {
             unsigned s;
             if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error;   /* overflow detection */
             do {
@@ -1472,11 +1526,14 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
         /* get offset */
         offset = LZ4_readLE16(ip); ip+=2;
         match = op - offset;
-        if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error;   /* Error : offset outside buffers */
-        LZ4_write32(op, (U32)offset);   /* costs ~1%; silence an msan warning when offset==0 */
 
         /* get matchlength */
         length = token & ML_MASK;
+
+_copy_match:
+        if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error;   /* Error : offset outside buffers */
+        LZ4_write32(op, (U32)offset);   /* costs ~1%; silence an msan warning when offset==0 */
+
         if (length == ML_MASK) {
             unsigned s;
             do {
-- 
cgit v0.12


From aaeeb2572bfa197bf29fd804953c93cba9db57be Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Sat, 28 Apr 2018 10:42:52 -0700
Subject: ignore windows+msys artefacts

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index 117b02d..829270b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,7 @@ bin/
 # Mac
 .DS_Store
 *.dSYM
+
+# Windows / Msys
+nul
+ld.exe*
-- 
cgit v0.12


From 5a2501a90d4a0fa581bce334fb23c2c4df69842c Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Sun, 29 Apr 2018 07:42:24 -0700
Subject: added a test case for LZ4_decompress_fast_usingDict

with a separated dictionary
since a joined dictionary is now detected as prefix64K.

Also : fixed a minor warning under msys
---
 programs/util.h   |  2 +-
 tests/fullbench.c | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/programs/util.h b/programs/util.h
index ef6ca77..d74db0d 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -194,7 +194,7 @@ extern "C" {
         return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
     }
 
-#elif (PLATFORM_POSIX_VERSION >= 200112L) && (defined __UCLIBC__ || ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) || __GLIBC__ > 2))
+#elif (PLATFORM_POSIX_VERSION >= 200112L) && (defined __UCLIBC__ || (defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) || __GLIBC__ > 2) ) )
 
     #include <time.h>
     typedef struct timespec UTIL_time_t;
diff --git a/tests/fullbench.c b/tests/fullbench.c
index ee2966f..c06e230 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -267,13 +267,20 @@ static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int
     return outSize;
 }
 
-static int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize)
+static int local_LZ4_decompress_fast_usingDict_prefix(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
     LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65536);
     return outSize;
 }
 
+static int local_LZ4_decompress_fast_usingExtDict(const char* in, char* out, int inSize, int outSize)
+{
+    (void)inSize;
+    LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65535);
+    return outSize;
+}
+
 static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
@@ -460,7 +467,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                 double averageTime;
                 clock_t clockTime;
 
-                PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize);
+                PROGRESS("%2i-%-34.34s :%10i ->\r", loopNb, compressorName, (int)benchedSize);
                 { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; }     /* warming up memory */
 
                 nb_loops = 0;
@@ -471,7 +478,8 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                     if (initFunction!=NULL) initFunction();
                     for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
                         chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
-                        if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
+                        if (chunkP[chunkNb].compressedSize==0)
+                            DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
                     }
                     nb_loops++;
                 }
@@ -482,13 +490,13 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                 if (averageTime < bestTime) bestTime = averageTime;
                 cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
                 ratio = (double)cSize/(double)benchedSize*100.;
-                PROGRESS("%1i- %-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
+                PROGRESS("%2i-%-34.34s :%10i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
             }
 
             if (ratio<100.)
-                DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
+                DISPLAY("%2i-%-34.34s :%10i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
             else
-                DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 100000);
+                DISPLAY("%2i-%-34.34s :%10i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 100000);
         }
 
         /* Prepare layout for decompression */
@@ -509,11 +517,12 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
         }
         for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
             chunkP[chunkNb].compressedSize = LZ4_compress_default(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize);
-            if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
+            if (chunkP[chunkNb].compressedSize==0)
+                DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
         }
 
         /* Decompression Algorithms */
-        for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (g_decompressionTest); dAlgNb++) {
+        for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && g_decompressionTest; dAlgNb++) {
             const char* dName;
             int (*decompressionFunction)(const char*, char*, int, int);
             double bestTime = 100000000.;
@@ -524,7 +533,8 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
             {
             case 0: DISPLAY("Decompression functions : \n"); continue;
             case 1: decompressionFunction = local_LZ4_decompress_fast; dName = "LZ4_decompress_fast"; break;
-            case 3: decompressionFunction = local_LZ4_decompress_fast_usingDict; dName = "LZ4_decompress_fast_usingDict"; break;
+            case 2: decompressionFunction = local_LZ4_decompress_fast_usingDict_prefix; dName = "LZ4_decompress_fast_usingDict(prefix)"; break;
+            case 3: decompressionFunction = local_LZ4_decompress_fast_usingExtDict; dName = "LZ4_decompress_fast_using(Ext)Dict"; break;
             case 4: decompressionFunction = LZ4_decompress_safe; dName = "LZ4_decompress_safe"; break;
             case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break;
             case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; break;
@@ -555,7 +565,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                 clock_t clockTime;
                 U32 crcDecoded;
 
-                PROGRESS("%1i- %-29.29s :%10i ->\r", loopNb, dName, (int)benchedSize);
+                PROGRESS("%2i-%-34.34s :%10i ->\r", loopNb, dName, (int)benchedSize);
 
                 nb_loops = 0;
                 clockTime = clock();
@@ -574,14 +584,14 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                 averageTime = (double)clockTime / nb_loops / CLOCKS_PER_SEC;
                 if (averageTime < bestTime) bestTime = averageTime;
 
-                PROGRESS("%1i- %-29.29s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
+                PROGRESS("%2i-%-34.34s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
 
                 /* CRC Checking */
                 crcDecoded = XXH32(orig_buff, (int)benchedSize, 0);
                 if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); }
             }
 
-            DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
+            DISPLAY("%2i-%-34.34s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
         }
       }
       free(orig_buff);
-- 
cgit v0.12


From e28ae0af078a632f0c66647396c69d4661364b79 Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Sun, 29 Apr 2018 08:46:39 -0700
Subject: updated NEWS for v1.8.2

mentioning work from @svpv
---
 NEWS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/NEWS b/NEWS
index c3b47f1..90cafc6 100644
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,7 @@ v1.8.2
 perf: *much* faster dictionary compression on small files, by @felixhandte
 perf: slightly faster HC compression and decompression speed
 perf: very small compression ratio improvement
+perf: improved decompression binary size and speed, by Alexey Tourbin (@svpv)
 fix : compression compatible with low memory addresses (< 0xFFFF)
 fix : decompression segfault when provided with NULL input, by @terrelln
 cli : new command --favor-decSpeed
-- 
cgit v0.12


From 4c696613a071bc31d111bed9f9bf85e392a80901 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 30 Apr 2018 15:55:33 -0700
Subject: clarified streaming decompression function

restrictions for ring buffer
---
 lib/lz4.h | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index db6353c..2745260 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -317,15 +317,19 @@ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const
  *  If less than 64KB of data has been decoded all the data must be present.
  *
  *  Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
- *  - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
- *    In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- *  - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
- *    maxBlockSize is implementation dependent. It's the maximum size of any single block.
+ *  - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+ *    maxBlockSize is the maximum size of any single block. It is implementation dependent, and can have any value (presumed > 16 bytes).
  *    In which case, encoding and decoding buffers do not need to be synchronized,
  *    and encoding ring buffer can have any size, including small ones ( < 64 KB).
- *  - _At least_ 64 KB + 8 bytes + maxBlockSize.
+ *  - Decompression buffer size is _at least_ 64 KB + 8 bytes + maxBlockSize.
  *    In which case, encoding and decoding buffers do not need to be synchronized,
  *    and encoding ring buffer can have any size, including larger than decoding buffer.
+ *  - Decompression buffer size is exactly the same as compression buffer size,
+ *    and follows exactly same update rule (block boundaries at same positions).
+ *    If the decoding function is provided with the exact decompressed size of each block,
+ *    then decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+ *    If the decoding function only knows the compressed size,
+ *    then buffer size must be a minimum of 64 KB + 8 bytes + maxBlockSize.
  *  Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
  *  and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
 */
-- 
cgit v0.12


From 8c574990a908233196080e70c12605f5f3d8eadd Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 30 Apr 2018 16:08:16 -0700
Subject: lz4hc changed variable

to reduce confusion
dictLowLimit => dictStart
---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f2dc21b..d42bb5a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -264,7 +264,7 @@ LZ4HC_InsertAndGetWiderMatch (
         } else {   /* matchIndex < dictLimit */
             const BYTE* const matchPtr = dictBase + matchIndex;
             if (LZ4_read32(matchPtr) == pattern) {
-                const BYTE* const dictLowLimit = dictBase + hc4->lowLimit;
+                const BYTE* const dictStart = dictBase + hc4->lowLimit;
                 int mlt;
                 int back = 0;
                 const BYTE* vLimit = ip + (dictLimit - matchIndex);
@@ -272,7 +272,7 @@ LZ4HC_InsertAndGetWiderMatch (
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
                 if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
                     mlt += LZ4_count(ip+mlt, lowPrefixPtr, iHighLimit);
-                back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictLowLimit) : 0;
+                back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0;
                 mlt -= back;
                 if (mlt > longest) {
                     longest = mlt;
-- 
cgit v0.12


From 1949bf11e3903ab6c615ac625587426dc11b4beb Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 30 Apr 2018 18:50:56 -0700
Subject: added visual test dir to .gitignore

---
 visual/.gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/visual/.gitignore b/visual/.gitignore
index dea92fc..276f8f5 100644
--- a/visual/.gitignore
+++ b/visual/.gitignore
@@ -6,5 +6,5 @@
 *.sdf
 *.suo
 *.user
-
+ver*/
 VS2010/bin/
-- 
cgit v0.12


From 6a7d501fed8444cb8a78cd55133916921bfbb07f Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 30 Apr 2018 18:56:16 -0700
Subject: renamed variable for clarity

lowLimit -> lowestMatchIndex
---
 lib/lz4hc.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d42bb5a..26295fc 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -223,7 +223,7 @@ LZ4HC_InsertAndGetWiderMatch (
     const U32 dictLimit = hc4->dictLimit;
     const BYTE* const lowPrefixPtr = base + dictLimit;
     const U32 ipIndex = (U32)(ip - base);
-    const U32 lowLimit = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
+    const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
     const BYTE* const dictBase = hc4->dictBase;
     int const lookBackLength = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
@@ -237,10 +237,10 @@ LZ4HC_InsertAndGetWiderMatch (
     /* First Match */
     LZ4HC_Insert(hc4, ip);
     matchIndex = HashTable[LZ4HC_hashPtr(ip)];
-    DEBUGLOG(7, "First match at index %u / %u (lowLimit)",
-                matchIndex, lowLimit);
+    DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)",
+                matchIndex, lowestMatchIndex);
 
-    while ((matchIndex>=lowLimit) && (nbAttempts)) {
+    while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
         DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
         nbAttempts--;
         assert(matchIndex < ipIndex);
@@ -308,13 +308,13 @@ LZ4HC_InsertAndGetWiderMatch (
                             matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
         }   }   }   }
-    }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
+    }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
 
-    if (dict == usingDictCtx && nbAttempts && ipIndex - lowLimit < MAX_DISTANCE) {
+    if (dict == usingDictCtx && nbAttempts && ipIndex - lowestMatchIndex < MAX_DISTANCE) {
         size_t const dictEndOffset = dictCtx->end - dictCtx->base;
         assert(dictEndOffset <= 1 GB);
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
-        matchIndex = dictMatchIndex + lowLimit - (U32)dictEndOffset;
+        matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset;
         while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
-- 
cgit v0.12


From 1a191b3f8d26b50a7c1d41590b529ec308d768cd Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 10:33:12 -0700
Subject: simplify shortcut

---
 lib/lz4.c | 77 ++++++++++++++++++---------------------------------------------
 1 file changed, 22 insertions(+), 55 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c6f0426..b46910f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1429,62 +1429,29 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
          */
         if ((endOnInput ? length != RUN_MASK : length <= 8) &&
             /* strictly "less than" on input, to re-enter the loop with at least one byte */
-            likely((endOnInput ? ip < shortiend : 1) && (op <= shortoend)))
+            likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)))
         {
-            /* Can we copy the literals with a single memcpy invocation?  Sometimes we can't
-             * copy 16 bytes, because they can clobber the dictionary in the ring buffer. */
-            if (!endOnInput /* only 8 bytes */ || /* nothing to clobber */ dict != usingExtDict) {
-                /* Copy the literals. */
-                memcpy(op, ip, endOnInput ? 16 : 8);
-                op += length; ip += length;
-
-                /* The second stage: prepare for match copying, decode full info.
-                 * It if doesn't work out, the info won't be wasted. */
-                length = token & ML_MASK; /* match length */
-                offset = LZ4_readLE16(ip); ip += 2;
-                match = op - offset;
-
-                /* Do not deal with overlapping matches. */
-                if ((length != 15) && (offset >= 8) &&
-                    (dict==withPrefix64k || match >= lowPrefix))
-                {
-                    /* Copy the match. */
-                    memcpy(op + 0, match + 0, 8);
-                    memcpy(op + 8, match + 8, 8);
-                    memcpy(op +16, match +16, 2);
-                    op += length + MINMATCH;
-                    /* Both stages worked, load the next token. */
-                    continue;
-                }
-            } else {
-                /* Save the literal length, can't copy 16 bytes just yet. */
-                size_t ll = length;
-
-                /* Prepare for the second satge. */
-                length = token & ML_MASK;
-                offset = LZ4_readLE16(ip+ll);
-                match = op + ll - offset;
-
-                if ((length != 15) && (offset >= 8) &&
-                    (dict==withPrefix64k || match >= lowPrefix))
-                {
-                    /* Copy the literals. */
-                    memcpy(op, ip, 16);
-                    op += ll; ip += ll + 2;
-                    /* Copy the match. */
-                    memcpy(op + 0, match + 0, 8);
-                    memcpy(op + 8, match + 8, 8);
-                    memcpy(op +16, match +16, 2);
-                    op += length + MINMATCH;
-                    /* Both stages worked, load the next token. */
-                    continue;
-                }
-
-                /* So we took the literlas, but the second stage didn't work. */
-                memcpy(op, ip, 8);
-                if (ll > 8)
-                    memcpy(op + 8, ip + 8, 8);
-                op += ll; ip += ll + 2;
+            /* Copy the literals. */
+            memcpy(op, ip, endOnInput ? 16 : 8);
+            op += length; ip += length;
+
+            /* The second stage: prepare for match copying, decode full info.
+             * If it doesn't work out, the info won't be wasted. */
+            length = token & ML_MASK; /* match length */
+            offset = LZ4_readLE16(ip); ip += 2;
+            match = op - offset;
+
+            /* Do not deal with overlapping matches. */
+            if ((length != 15) && (offset >= 8) &&
+                (dict==withPrefix64k || match >= lowPrefix))
+            {
+                /* Copy the match. */
+                memcpy(op + 0, match + 0, 8);
+                memcpy(op + 8, match + 8, 8);
+                memcpy(op +16, match +16, 2);
+                op += length + MINMATCH;
+                /* Both stages worked, load the next token. */
+                continue;
             }
 
             /* The second stage didn't work out, but the info is ready.
-- 
cgit v0.12


From 93cf628a08e82256abc77d1fd144bda78b7ee1df Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 12:56:37 -0700
Subject: introduce LZ4_decoderRingBufferSize()

fuzzer : fix and robustify ring buffer tests
---
 lib/lz4.c      | 33 +++++++++++++++++-----
 lib/lz4.h      | 56 +++++++++++++++++++++++-------------
 tests/fuzzer.c | 89 ++++++++++++++++++++++++++++++++++------------------------
 3 files changed, 116 insertions(+), 62 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index b46910f..71fe8f3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1431,7 +1431,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             /* strictly "less than" on input, to re-enter the loop with at least one byte */
             likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)))
         {
-            /* Copy the literals. */
+            /* Copy the literals */
             memcpy(op, ip, endOnInput ? 16 : 8);
             op += length; ip += length;
 
@@ -1688,12 +1688,11 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
     return 0;
 }
 
-/*!
- * LZ4_setStreamDecode() :
- * Use this function to instruct where to find the dictionary.
- * This function is not necessary if previous data is still available where it was decoded.
- * Loading a size of 0 is allowed (same effect as no dictionary).
- * Return : 1 if OK, 0 if error
+/*! LZ4_setStreamDecode() :
+ *  Use this function to instruct where to find the dictionary.
+ *  This function is not necessary if previous data is still available where it was decoded.
+ *  Loading a size of 0 is allowed (same effect as no dictionary).
+ * @return : 1 if OK, 0 if error
  */
 int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize)
 {
@@ -1705,6 +1704,26 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
     return 1;
 }
 
+/*! LZ4_decoderRingBufferSize() :
+ *  when setting a ring buffer for streaming decompression (optional scenario),
+ *  provides the minimum size of this ring buffer
+ *  to be compatible with any source respecting maxBlockSize condition.
+ *  Note : in a ring buffer scenario,
+ *  blocks are presumed decompressed next to each other
+ *  up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
+ *  at which stage it resumes from beginning of ring buffer.
+ * @return : minimum ring buffer size,
+ *           or 0 if there is an error (invalid maxBlockSize).
+ */
+
+int LZ4_decoderRingBufferSize(int maxBlockSize)
+{
+    if (maxBlockSize < 0) return 0;
+    if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0;
+    if (maxBlockSize < 16) maxBlockSize = 16;
+    return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize);
+}
+
 /*
 *_continue() :
     These decoding functions allow decompression of multiple blocks in "streaming" mode.
diff --git a/lib/lz4.h b/lib/lz4.h
index 2745260..410f480 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -293,45 +293,62 @@ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxD
 *  Streaming Decompression Functions
 *  Bufferless synchronous API
 ************************************************/
-typedef union LZ4_streamDecode_u LZ4_streamDecode_t;   /* incomplete type (defined later) */
+typedef union LZ4_streamDecode_u LZ4_streamDecode_t;   /* tracking context */
 
 /*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() :
- *  creation / destruction of streaming decompression tracking structure.
- *  A tracking structure can be re-used multiple times sequentially. */
+ *  creation / destruction of streaming decompression tracking context.
+ *  A tracking context can be re-used multiple times.
+ */
 LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
 LZ4LIB_API int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
 
 /*! LZ4_setStreamDecode() :
- *  An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
+ *  An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
  *  Use this function to start decompression of a new stream of blocks.
  *  A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
+ *  Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
  * @return : 1 if OK, 0 if error
  */
 LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
 
+/*! LZ4_decoderRingBufferSize() : v1.8.2
+ *  Note : in a ring buffer scenario (optional),
+ *  blocks are presumed decompressed next to each other
+ *  up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
+ *  at which stage it resumes from beginning of ring buffer.
+ *  When setting such a ring buffer for streaming decompression,
+ *  provides the minimum size of this ring buffer
+ *  to be compatible with any source respecting maxBlockSize condition.
+ * @return : minimum ring buffer size,
+ *           or 0 if there is an error (invalid maxBlockSize).
+ */
+LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize);
+#define LZ4_DECODER_RING_BUFFER_SIZE(mbs) (65536 + 14 + (mbs))  /* for static allocation; mbs presumed valid */
+
 /*! LZ4_decompress_*_continue() :
  *  These decoding functions allow decompression of consecutive blocks in "streaming" mode.
  *  A block is an unsplittable entity, it must be presented entirely to a decompression function.
- *  Decompression functions only accept one block at a time.
+ *  Decompression functions only accepts one block at a time.
  *  The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded.
- *  If less than 64KB of data has been decoded all the data must be present.
+ *  If less than 64KB of data has been decoded, all the data must be present.
  *
- *  Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
+ *  Special : if decompression side sets a ring buffer, it must respect one of the following conditions :
+ *  - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize).
+ *    maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes.
+ *    In which case, encoding and decoding buffers do not need to be synchronized.
+ *    Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize.
+ *  - Synchronized mode :
+ *    Decompression buffer size is _exactly_ the same as compression buffer size,
+ *    and follows exactly same update rule (block boundaries at same positions),
+ *    and decoding function is provided with exact decompressed size of each block (exception for last block of the stream),
+ *    _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB).
  *  - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes.
- *    maxBlockSize is the maximum size of any single block. It is implementation dependent, and can have any value (presumed > 16 bytes).
  *    In which case, encoding and decoding buffers do not need to be synchronized,
  *    and encoding ring buffer can have any size, including small ones ( < 64 KB).
- *  - Decompression buffer size is _at least_ 64 KB + 8 bytes + maxBlockSize.
- *    In which case, encoding and decoding buffers do not need to be synchronized,
- *    and encoding ring buffer can have any size, including larger than decoding buffer.
- *  - Decompression buffer size is exactly the same as compression buffer size,
- *    and follows exactly same update rule (block boundaries at same positions).
- *    If the decoding function is provided with the exact decompressed size of each block,
- *    then decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- *    If the decoding function only knows the compressed size,
- *    then buffer size must be a minimum of 64 KB + 8 bytes + maxBlockSize.
- *  Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
- *  and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
+ *
+ *  Whenever these conditions are not possible,
+ *  save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,
+ *  then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
 */
 LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
 LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
@@ -341,6 +358,7 @@ LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecod
  *  These decoding functions work the same as
  *  a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
  *  They are stand-alone, and don't need an LZ4_streamDecode_t structure.
+ *  Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
  */
 LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
 LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 1fbda8a..28f14bf 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -47,6 +47,7 @@
 #include <stdio.h>      /* fgets, sscanf */
 #include <string.h>     /* strcmp */
 #include <time.h>       /* clock_t, clock, CLOCKS_PER_SEC */
+#include <assert.h>
 #define LZ4_STATIC_LINKING_ONLY
 #define LZ4_HC_STATIC_LINKING_ONLY
 #include "lz4hc.h"
@@ -953,6 +954,7 @@ static void FUZ_unitTests(int compressionLevel)
     const unsigned cycleNb= 0;
     char testInput[testInputSize];
     char testCompressed[testCompressedSize];
+    size_t const testVerifySize = testInputSize;
     char testVerify[testInputSize];
     char ringBuffer[ringBufferSize];
     U32 randState = 1;
@@ -1197,19 +1199,28 @@ static void FUZ_unitTests(int compressionLevel)
             }
         }
 
-        /* small decoder-side ring buffer test */
+        /* Ring buffer test : Non synchronized decoder */
+        /* This test uses minimum amount of memory required to setup a decoding ring buffer
+         * while being unsynchronized with encoder
+         * (no assumption done on how the data is encoded, it just follows LZ4 format specification).
+         * This size is documented in lz4.h, and is LZ4_decoderRingBufferSize(maxBlockSize).
+         */
         {   XXH64_state_t xxhOrig;
             XXH64_state_t xxhNewSafe, xxhNewFast;
             LZ4_streamDecode_t decodeStateSafe, decodeStateFast;
-            const U32 maxMessageSizeLog = 12;
-            const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
-            U32 messageSize;
+            const int maxMessageSizeLog = 12;
+            const int maxMessageSize = 1 << maxMessageSizeLog;
+            const int maxMessageSizeMask = maxMessageSize - 1;
+            int messageSize;
             U32 totalMessageSize = 0;
-            U32 iNext = 0;
-            U32 dNext = 0;
-            const U32 dBufferSize = 64 KB;
+            const int dBufferSize = LZ4_decoderRingBufferSize(maxMessageSize);
+            char* const ringBufferSafe = testVerify;
+            char* const ringBufferFast = testVerify + dBufferSize + 1;   /* used by LZ4_decompress_fast_continue */
+            int iNext = 0;
+            int dNext = 0;
             int compressedSize;
 
+            assert((size_t)(dBufferSize + 1 + dBufferSize) < testVerifySize);   /* space used by ringBufferSafe and ringBufferFast */
             XXH64_reset(&xxhOrig, 0);
             XXH64_reset(&xxhNewSafe, 0);
             XXH64_reset(&xxhNewFast, 0);
@@ -1217,38 +1228,40 @@ static void FUZ_unitTests(int compressionLevel)
             LZ4_setStreamDecode(&decodeStateSafe, NULL, 0);
             LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
 
-#define BSIZE1 65537
-#define BSIZE2 16435
+#define BSIZE1 (dBufferSize - (maxMessageSize-1))
 
             /* first block */
-            messageSize = BSIZE1;
+            messageSize = BSIZE1;   /* note : we cheat a bit here, in theory no message should be > maxMessageSize. We just want to fill the decoding ring buffer once. */
+            assert(messageSize < dBufferSize);
             XXH64_update(&xxhOrig, testInput + iNext, messageSize);
             crcOrig = XXH64_digest(&xxhOrig);
 
             compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
             FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
 
-            result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
-            FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
+            result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, ringBufferSafe + dNext, compressedSize, messageSize);
+            FUZ_CHECKTEST(result!=messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
 
-            XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+            XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize);
             { U64 const crcNew = XXH64_digest(&xxhNewSafe);
               FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
 
-            result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+            result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize);
             FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed");
 
-            XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+            XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize);
             { U64 const crcNew = XXH64_digest(&xxhNewFast);
               FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
 
-            /* prepare next message */
+            /* prepare second message */
             dNext += messageSize;
             totalMessageSize += messageSize;
-            messageSize = BSIZE2;
-            iNext = 132000;
-            memcpy(testInput + iNext, testInput + 8, messageSize);
-            if (dNext > dBufferSize) dNext = 0;
+            messageSize = maxMessageSize;
+            iNext = BSIZE1+1;
+            assert(BSIZE1 >= 65535);
+            memcpy(testInput + iNext, testInput + (BSIZE1-65535), messageSize);  /* will generate a match at max distance == 65535 */
+            FUZ_CHECKTEST(dNext+messageSize <= dBufferSize, "Ring buffer test : second message should require restarting from beginning");
+            dNext = 0;
 
             while (totalMessageSize < 9 MB) {
                 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
@@ -1256,32 +1269,36 @@ static void FUZ_unitTests(int compressionLevel)
 
                 compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
                 FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
-
-#if 1           /* Because the ring buffer is small, decompression overwrites part of the output which
-                 * is first used as dictionary.  Hence only one decompression function can be tested. */
-                result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
-                FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
-                XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+                DISPLAYLEVEL(5, "compressed %i bytes to %i bytes \n", messageSize, compressedSize);
+
+                /* test LZ4_decompress_safe_continue */
+                assert(dNext < dBufferSize);
+                assert(dBufferSize - dNext >= maxMessageSize);
+                result = LZ4_decompress_safe_continue(&decodeStateSafe,
+                                                      testCompressed, ringBufferSafe + dNext,
+                                                      compressedSize, dBufferSize - dNext);   /* works without knowing messageSize, but under assumption that messageSize < maxMessageSize */
+                FUZ_CHECKTEST(result!=messageSize, "D.ringBuffer : LZ4_decompress_safe_continue() test failed");
+                XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize);
                 {   U64 const crcNew = XXH64_digest(&xxhNewSafe);
-                    if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, testVerify + dNext);
-                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during small decoder-side ring buffer test");
+                    if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferSafe + dNext);
+                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during D.ringBuffer test");
                 }
-#else
-                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
-                FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed");
 
-                XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+                /* test LZ4_decompress_fast_continue in its own buffer ringBufferFast */
+                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize);
+                FUZ_CHECKTEST(result!=compressedSize, "D.ringBuffer : LZ4_decompress_fast_continue() test failed");
+                XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize);
                 {   U64 const crcNew = XXH64_digest(&xxhNewFast);
-                    if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, testVerify + dNext);
-                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during small decoder-side ring buffer test");
+                    if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferFast + dNext);
+                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during D.ringBuffer test");
                 }
-#endif
+
                 /* prepare next message */
                 dNext += messageSize;
                 totalMessageSize += messageSize;
                 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
                 iNext = (FUZ_rand(&randState) & 65535);
-                if (dNext > dBufferSize) dNext = 0;
+                if (dNext + maxMessageSize > dBufferSize) dNext = 0;
             }
         }
     }
-- 
cgit v0.12


From 999a8488f60cb076944d69d2aea9828f890a4e43 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 13:57:33 -0700
Subject: removed test that might be optimized away

under UB rule "no overflow on int"
---
 tests/fuzzer.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 28f14bf..5dd75b3 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1232,7 +1232,6 @@ static void FUZ_unitTests(int compressionLevel)
 
             /* first block */
             messageSize = BSIZE1;   /* note : we cheat a bit here, in theory no message should be > maxMessageSize. We just want to fill the decoding ring buffer once. */
-            assert(messageSize < dBufferSize);
             XXH64_update(&xxhOrig, testInput + iNext, messageSize);
             crcOrig = XXH64_digest(&xxhOrig);
 
@@ -1276,7 +1275,7 @@ static void FUZ_unitTests(int compressionLevel)
                 assert(dBufferSize - dNext >= maxMessageSize);
                 result = LZ4_decompress_safe_continue(&decodeStateSafe,
                                                       testCompressed, ringBufferSafe + dNext,
-                                                      compressedSize, dBufferSize - dNext);   /* works without knowing messageSize, but under assumption that messageSize < maxMessageSize */
+                                                      compressedSize, dBufferSize - dNext);   /* works without knowing messageSize, under assumption that messageSize <= maxMessageSize */
                 FUZ_CHECKTEST(result!=messageSize, "D.ringBuffer : LZ4_decompress_safe_continue() test failed");
                 XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize);
                 {   U64 const crcNew = XXH64_digest(&xxhNewSafe);
-- 
cgit v0.12


From 85be6b8f6d5a91a8834e913b5f547ded49f7f714 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 14:22:35 -0700
Subject: increased nbAttempts for lz4 -12

shaves one more kilobyte from silesia.tar
---
 lib/lz4hc.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index bb1b1a6..0859ea6 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -681,19 +681,19 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
         U32 targetLength;
     } cParams_t;
     static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = {
-        { lz4hc,    2, 16 },  /* 0, unused */
-        { lz4hc,    2, 16 },  /* 1, unused */
-        { lz4hc,    2, 16 },  /* 2, unused */
-        { lz4hc,    4, 16 },  /* 3 */
-        { lz4hc,    8, 16 },  /* 4 */
-        { lz4hc,   16, 16 },  /* 5 */
-        { lz4hc,   32, 16 },  /* 6 */
-        { lz4hc,   64, 16 },  /* 7 */
-        { lz4hc,  128, 16 },  /* 8 */
-        { lz4hc,  256, 16 },  /* 9 */
-        { lz4opt,  96, 64 },  /*10==LZ4HC_CLEVEL_OPT_MIN*/
-        { lz4opt, 512,128 },  /*11 */
-        { lz4opt,8192, LZ4_OPT_NUM },  /* 12==LZ4HC_CLEVEL_MAX */
+        { lz4hc,     2, 16 },  /* 0, unused */
+        { lz4hc,     2, 16 },  /* 1, unused */
+        { lz4hc,     2, 16 },  /* 2, unused */
+        { lz4hc,     4, 16 },  /* 3 */
+        { lz4hc,     8, 16 },  /* 4 */
+        { lz4hc,    16, 16 },  /* 5 */
+        { lz4hc,    32, 16 },  /* 6 */
+        { lz4hc,    64, 16 },  /* 7 */
+        { lz4hc,   128, 16 },  /* 8 */
+        { lz4hc,   256, 16 },  /* 9 */
+        { lz4opt,   96, 64 },  /*10==LZ4HC_CLEVEL_OPT_MIN*/
+        { lz4opt,  512,128 },  /*11 */
+        { lz4opt,16384,LZ4_OPT_NUM },  /* 12==LZ4HC_CLEVEL_MAX */
     };
 
     DEBUGLOG(4, "LZ4HC_compress_generic(%p, %p, %d)", ctx, src, *srcSizePtr);
-- 
cgit v0.12


From c25eb1666654c378a6f9e74c9975181cb2767a2a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 16:05:42 -0700
Subject: random lz4f clarifications

the initial intention was to update lz4f ring buffer strategy,
but lz4f doesn't use ring buffer.
Instead, it uses the destination buffer as much as possible,
and merely copies just what's required to preserve history
into its own buffer, at the end.
Pretty efficient.

This patch just clarifies a few comments and add some assert().
It's built on top of #528.

It also updates doc.
---
 doc/lz4_manual.html | 54 ++++++++++++++++++++++++++-----------
 lib/lz4frame.c      | 76 +++++++++++++++++++++++++++++++++--------------------
 2 files changed, 85 insertions(+), 45 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index ddd2724..e079db2 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -206,38 +206,59 @@ int           LZ4_freeStream (LZ4_stream_t* streamPtr);
 
 <pre><b>LZ4_streamDecode_t* LZ4_createStreamDecode(void);
 int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
-</b><p>  creation / destruction of streaming decompression tracking structure.
-  A tracking structure can be re-used multiple times sequentially. 
+</b><p>  creation / destruction of streaming decompression tracking context.
+  A tracking context can be re-used multiple times.
+ 
 </p></pre><BR>
 
 <pre><b>int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
-</b><p>  An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
+</b><p>  An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
   Use this function to start decompression of a new stream of blocks.
   A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
+  Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
  @return : 1 if OK, 0 if error
  
 </p></pre><BR>
 
+<pre><b>int LZ4_decoderRingBufferSize(int maxBlockSize);
+#define LZ4_DECODER_RING_BUFFER_SIZE(mbs) (65536 + 14 + (mbs))  </b>/* for static allocation; mbs presumed valid */<b>
+</b><p>  Note : in a ring buffer scenario (optional),
+  blocks are presumed decompressed next to each other
+  up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
+  at which stage it resumes from beginning of ring buffer.
+  When setting such a ring buffer for streaming decompression,
+  provides the minimum size of this ring buffer
+  to be compatible with any source respecting maxBlockSize condition.
+ @return : minimum ring buffer size,
+           or 0 if there is an error (invalid maxBlockSize).
+ 
+</p></pre><BR>
+
 <pre><b>int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
 </b><p>  These decoding functions allow decompression of consecutive blocks in "streaming" mode.
   A block is an unsplittable entity, it must be presented entirely to a decompression function.
-  Decompression functions only accept one block at a time.
+  Decompression functions only accepts one block at a time.
   The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded.
-  If less than 64KB of data has been decoded all the data must be present.
-
-  Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
-  - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
-    In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
-  - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
-    maxBlockSize is implementation dependent. It's the maximum size of any single block.
+  If less than 64KB of data has been decoded, all the data must be present.
+
+  Special : if decompression side sets a ring buffer, it must respect one of the following conditions :
+  - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize).
+    maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes.
+    In which case, encoding and decoding buffers do not need to be synchronized.
+    Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize.
+  - Synchronized mode :
+    Decompression buffer size is _exactly_ the same as compression buffer size,
+    and follows exactly same update rule (block boundaries at same positions),
+    and decoding function is provided with exact decompressed size of each block (exception for last block of the stream),
+    _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB).
+  - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes.
     In which case, encoding and decoding buffers do not need to be synchronized,
     and encoding ring buffer can have any size, including small ones ( < 64 KB).
-  - _At least_ 64 KB + 8 bytes + maxBlockSize.
-    In which case, encoding and decoding buffers do not need to be synchronized,
-    and encoding ring buffer can have any size, including larger than decoding buffer.
-  Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
-  and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
+
+  Whenever these conditions are not possible,
+  save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,
+  then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
 </p></pre><BR>
 
 <pre><b>int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
@@ -245,6 +266,7 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
 </b><p>  These decoding functions work the same as
   a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
   They are stand-alone, and don't need an LZ4_streamDecode_t structure.
+  Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
  
 </p></pre><BR>
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 6cf2a06..f57db24 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -96,6 +96,19 @@ You can contact the author at :
 
 #define LZ4F_STATIC_ASSERT(c)    { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) && !defined(DEBUGLOG)
+#  include <stdio.h>
+static int g_debuglog_enable = 1;
+#  define DEBUGLOG(l, ...) {                                  \
+                if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
+                    fprintf(stderr, __FILE__ ": ");           \
+                    fprintf(stderr, __VA_ARGS__);             \
+                    fprintf(stderr, " \n");                   \
+            }   }
+#else
+#  define DEBUGLOG(l, ...)      {}    /* disabled */
+#endif
+
 
 /*-************************************
 *  Basic Types
@@ -408,6 +421,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     LZ4_stream_t lz4ctx;
     LZ4F_cctx_t *cctxPtr = &cctx;
 
+    DEBUGLOG(4, "LZ4F_compressFrame");
     MEM_INIT(&cctx, 0, sizeof(cctx));
     cctx.version = LZ4F_VERSION;
     cctx.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
@@ -1198,24 +1212,31 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoP
 
 /* LZ4F_updateDict() :
  * only used for LZ4F_blockLinked mode */
-static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
+static void LZ4F_updateDict(LZ4F_dctx* dctx,
+                      const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart,
+                      unsigned withinTmp)
 {
     if (dctx->dictSize==0)
         dctx->dict = (const BYTE*)dstPtr;   /* priority to dictionary continuity */
 
-    if (dctx->dict + dctx->dictSize == dstPtr) {  /* dictionary continuity */
+    if (dctx->dict + dctx->dictSize == dstPtr) {  /* dictionary continuity, directly within dstBuffer */
         dctx->dictSize += dstSize;
         return;
     }
 
-    if (dstPtr - dstPtr0 + dstSize >= 64 KB) {  /* dstBuffer large enough to become dictionary */
-        dctx->dict = (const BYTE*)dstPtr0;
-        dctx->dictSize = dstPtr - dstPtr0 + dstSize;
+    if (dstPtr - dstBufferStart + dstSize >= 64 KB) {  /* history in dstBuffer becomes large enough to become dictionary */
+        dctx->dict = (const BYTE*)dstBufferStart;
+        dctx->dictSize = dstPtr - dstBufferStart + dstSize;
         return;
     }
 
-    if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) {
-        /* assumption : dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart */
+    assert(dstSize < 64 KB);   /* if dstSize >= 64 KB, dictionary would be set into dstBuffer directly */
+
+    /* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOut */
+
+    if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) {   /* continue history within tmpOutBuffer */
+        /* withinTmp expectation : content of [dstPtr,dstSize] is same as [dict+dictSize,dstSize], so we just extend it */
+        assert(dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart);
         dctx->dictSize += dstSize;
         return;
     }
@@ -1236,7 +1257,7 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize,
 
     if (dctx->dict == dctx->tmpOutBuffer) {    /* copy dst into tmp to complete dict */
         if (dctx->dictSize + dstSize > dctx->maxBufferSize) {  /* tmp buffer not large enough */
-            size_t const preserveSize = 64 KB - dstSize;   /* note : dstSize < 64 KB */
+            size_t const preserveSize = 64 KB - dstSize;
             memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
             dctx->dictSize = preserveSize;
         }
@@ -1246,7 +1267,7 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize,
     }
 
     /* join dict & dest into tmp */
-    {   size_t preserveSize = 64 KB - dstSize;   /* note : dstSize < 64 KB */
+    {   size_t preserveSize = 64 KB - dstSize;
         if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize;
         memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
         memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize);
@@ -1313,7 +1334,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             }
             dctx->tmpInSize = 0;
             if (srcEnd-srcPtr == 0) return minFHSize;   /* 0-size input */
-            dctx->tmpInTarget = minFHSize;   /* minimum to attempt decode */
+            dctx->tmpInTarget = minFHSize;   /* minimum size to decode header */
             dctx->dStage = dstage_storeFrameHeader;
             /* fall-through */
 
@@ -1470,8 +1491,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                     U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
                     if (readCRC != calcCRC)
                         return err0r(LZ4F_ERROR_blockChecksum_invalid);
-                }
-            }
+            }   }
             dctx->dStage = dstage_getBlockHeader;  /* new block */
             break;
 
@@ -1512,13 +1532,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             }   }
 
             if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {
-                const char *dict = (const char *)dctx->dict;
+                const char* dict = (const char*)dctx->dict;
                 size_t dictSize = dctx->dictSize;
                 int decodedSize;
                 if (dict && dictSize > 1 GB) {
                     /* the dictSize param is an int, avoid truncation / sign issues */
-                    dict += dictSize - 1 GB;
-                    dictSize = 1 GB;
+                    dict += dictSize - 64 KB;
+                    dictSize = 64 KB;
                 }
                 /* enough capacity in `dst` to decompress directly there */
                 decodedSize = LZ4_decompress_safe_usingDict(
@@ -1552,18 +1572,16 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 } else {  /* dict not within tmp */
                     size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
                     dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
-                }
-            }
+            }   }
 
             /* Decode block */
-            {
-                const char *dict = (const char *)dctx->dict;
+            {   const char* dict = (const char*)dctx->dict;
                 size_t dictSize = dctx->dictSize;
                 int decodedSize;
                 if (dict && dictSize > 1 GB) {
                     /* the dictSize param is an int, avoid truncation / sign issues */
-                    dict += dictSize - 1 GB;
-                    dictSize = 1 GB;
+                    dict += dictSize - 64 KB;
+                    dictSize = 64 KB;
                 }
                 decodedSize = LZ4_decompress_safe_usingDict(
                         (const char*)selectedIn, (char*)dctx->tmpOut,
@@ -1586,8 +1604,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy);
 
                 /* dictionary management */
-                if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
-                    LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1);
+                if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
+                    LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1 /*withinTmp*/);
 
                 dctx->tmpOutStart += sizeToCopy;
                 dstPtr += sizeToCopy;
@@ -1596,8 +1614,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                     dctx->dStage = dstage_getBlockHeader;  /* get next block */
                     break;
                 }
+                /* could not flush everything : stop there, just request a block header */
+                doAnotherStage = 0;
                 nextSrcSizeHint = BHSize;
-                doAnotherStage = 0;   /* still some data to flush */
                 break;
             }
 
@@ -1634,7 +1653,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 selectedIn = dctx->tmpIn;
             }   /* if (dctx->dStage == dstage_storeSuffix) */
 
-        /* case dstage_checkSuffix: */   /* no direct call, avoid scan-build warning */
+        /* case dstage_checkSuffix: */   /* no direct entry, avoid initialization risks */
             {   U32 const readCRC = LZ4F_readLE32(selectedIn);
                 U32 const resultCRC = XXH32_digest(&(dctx->xxh));
                 if (readCRC != resultCRC)
@@ -1658,8 +1677,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
 
             if (dctx->dStage == dstage_storeSFrameSize)
         case dstage_storeSFrameSize:
-            {
-                size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
+            {   size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
                                              (size_t)(srcEnd - srcPtr) );
                 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
                 srcPtr += sizeToCopy;
@@ -1673,7 +1691,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 selectedIn = dctx->header + 4;
             }   /* if (dctx->dStage == dstage_storeSFrameSize) */
 
-        /* case dstage_decodeSFrameSize: */   /* no direct access */
+        /* case dstage_decodeSFrameSize: */   /* no direct entry */
             {   size_t const SFrameSize = LZ4F_readLE32(selectedIn);
                 dctx->frameInfo.contentSize = SFrameSize;
                 dctx->tmpInTarget = SFrameSize;
@@ -1692,7 +1710,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 LZ4F_resetDecompressionContext(dctx);
                 break;
             }
-        }
+        }   /* switch (dctx->dStage) */
     }   /* while (doAnotherStage) */
 
     /* preserve history within tmp whenever necessary */
-- 
cgit v0.12


From 543223851fe4b334ff6380d71d4ca94c822fcc8a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 16:41:15 -0700
Subject: updated benchmark for v1.8.2

---
 README.md | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index 596fdac..5d0aa48 100644
--- a/README.md
+++ b/README.md
@@ -43,8 +43,8 @@ Benchmarks
 -------------------------
 
 The benchmark uses [lzbench], from @inikep
-compiled with GCC v6.2.0 on Linux 64-bits.
-The reference system uses a Core i7-3930K CPU @ 4.5GHz.
+compiled with GCC v7.3.0 on Linux 64-bits (Debian 4.15.17-1).
+The reference system uses a Core i7-6700K CPU @ 4.0GHz.
 Benchmark evaluates the compression of reference [Silesia Corpus]
 in single-thread mode.
 
@@ -53,23 +53,22 @@ in single-thread mode.
 
 |  Compressor            | Ratio   | Compression | Decompression |
 |  ----------            | -----   | ----------- | ------------- |
-|  memcpy                |  1.000  | 7300 MB/s   |   7300 MB/s   |
-|**LZ4 fast 8  (v1.7.3)**|  1.799  |**911 MB/s** | **3360 MB/s** |
-|**LZ4 default (v1.7.3)**|**2.101**|**625 MB/s** | **3220 MB/s** |
-|  LZO 2.09              |  2.108  |  620 MB/s   |    845 MB/s   |
-|  QuickLZ 1.5.0         |  2.238  |  510 MB/s   |    600 MB/s   |
-|  Snappy 1.1.3          |  2.091  |  450 MB/s   |   1550 MB/s   |
-|  LZF v3.6              |  2.073  |  365 MB/s   |    820 MB/s   |
-|  [Zstandard] 1.1.1 -1  |  2.876  |  330 MB/s   |    930 MB/s   |
-|  [Zstandard] 1.1.1 -3  |  3.164  |  200 MB/s   |    810 MB/s   |
-| [zlib] deflate 1.2.8 -1|  2.730  |  100 MB/s   |    370 MB/s   |
-|**LZ4 HC -9 (v1.7.3)**  |**2.720**|   34 MB/s   | **3240 MB/s** |
-| [zlib] deflate 1.2.8 -6|  3.099  |   33 MB/s   |    390 MB/s   |
+|  memcpy                |  1.000  |13100 MB/s   |  13100 MB/s   |
+|**LZ4 default (v1.8.2)**|**2.101**|**730 MB/s** | **3900 MB/s** |
+|  LZO 2.09              |  2.108  |  630 MB/s   |    800 MB/s   |
+|  QuickLZ 1.5.0         |  2.238  |  530 MB/s   |    720 MB/s   |
+|  Snappy 1.1.4          |  2.091  |  525 MB/s   |   1750 MB/s   |
+|  [Zstandard] 1.3.4 -1  |  2.877  |  470 MB/s   |   1380 MB/s   |
+|  LZF v3.6              |  2.073  |  380 MB/s   |    840 MB/s   |
+| [zlib] deflate 1.2.8 -1|  2.730  |  100 MB/s   |    380 MB/s   |
+|**LZ4 HC -9 (v1.8.2)**  |**2.721**|   40 MB/s   | **3920 MB/s** |
+| [zlib] deflate 1.2.8 -6|  3.099  |   34 MB/s   |    410 MB/s   |
 
 [zlib]: http://www.zlib.net/
 [Zstandard]: http://www.zstd.net/
 
-LZ4 is also compatible and well optimized for x32 mode, for which it provides an additional +10% speed performance.
+LZ4 is also compatible and well optimized for x32 mode,
+for which it provides some additional speed performance.
 
 
 Installation
-- 
cgit v0.12


From ba168ee5c7f9e641c54a9a9a2668a0acec153121 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 16:41:15 -0700
Subject: updated benchmark for v1.8.2

---
 README.md | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 596fdac..406792a 100644
--- a/README.md
+++ b/README.md
@@ -43,33 +43,32 @@ Benchmarks
 -------------------------
 
 The benchmark uses [lzbench], from @inikep
-compiled with GCC v6.2.0 on Linux 64-bits.
-The reference system uses a Core i7-3930K CPU @ 4.5GHz.
+compiled with GCC v7.3.0 on Linux 64-bits (Debian 4.15.17-1).
+The reference system uses a Core i7-6700K CPU @ 4.0GHz.
 Benchmark evaluates the compression of reference [Silesia Corpus]
 in single-thread mode.
 
 [lzbench]: https://github.com/inikep/lzbench
 [Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
 
-|  Compressor            | Ratio   | Compression | Decompression |
-|  ----------            | -----   | ----------- | ------------- |
-|  memcpy                |  1.000  | 7300 MB/s   |   7300 MB/s   |
-|**LZ4 fast 8  (v1.7.3)**|  1.799  |**911 MB/s** | **3360 MB/s** |
-|**LZ4 default (v1.7.3)**|**2.101**|**625 MB/s** | **3220 MB/s** |
-|  LZO 2.09              |  2.108  |  620 MB/s   |    845 MB/s   |
-|  QuickLZ 1.5.0         |  2.238  |  510 MB/s   |    600 MB/s   |
-|  Snappy 1.1.3          |  2.091  |  450 MB/s   |   1550 MB/s   |
-|  LZF v3.6              |  2.073  |  365 MB/s   |    820 MB/s   |
-|  [Zstandard] 1.1.1 -1  |  2.876  |  330 MB/s   |    930 MB/s   |
-|  [Zstandard] 1.1.1 -3  |  3.164  |  200 MB/s   |    810 MB/s   |
-| [zlib] deflate 1.2.8 -1|  2.730  |  100 MB/s   |    370 MB/s   |
-|**LZ4 HC -9 (v1.7.3)**  |**2.720**|   34 MB/s   | **3240 MB/s** |
-| [zlib] deflate 1.2.8 -6|  3.099  |   33 MB/s   |    390 MB/s   |
+|  Compressor             | Ratio   | Compression | Decompression |
+|  ----------             | -----   | ----------- | ------------- |
+|  memcpy                 |  1.000  |13100 MB/s   |  13100 MB/s   |
+|**LZ4 default (v1.8.2)** |**2.101**|**730 MB/s** | **3900 MB/s** |
+|  LZO 2.09               |  2.108  |  630 MB/s   |    800 MB/s   |
+|  QuickLZ 1.5.0          |  2.238  |  530 MB/s   |    720 MB/s   |
+|  Snappy 1.1.4           |  2.091  |  525 MB/s   |   1750 MB/s   |
+|  [Zstandard] 1.3.4 -1   |  2.877  |  470 MB/s   |   1380 MB/s   |
+|  LZF v3.6               |  2.073  |  380 MB/s   |    840 MB/s   |
+| [zlib] deflate 1.2.11 -1|  2.730  |  100 MB/s   |    380 MB/s   |
+|**LZ4 HC -9 (v1.8.2)**   |**2.721**|   40 MB/s   | **3920 MB/s** |
+| [zlib] deflate 1.2.11 -6|  3.099  |   34 MB/s   |    410 MB/s   |
 
 [zlib]: http://www.zlib.net/
 [Zstandard]: http://www.zstd.net/
 
-LZ4 is also compatible and well optimized for x32 mode, for which it provides an additional +10% speed performance.
+LZ4 is also compatible and well optimized for x32 mode,
+for which it provides some additional speed performance.
 
 
 Installation
-- 
cgit v0.12


From 1e130d23e3b5f86e30e5dd7d3c1fc6d93d63e53f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 16:52:33 -0700
Subject: updated NEWS in preparation for v1.8.2

---
 NEWS | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 90cafc6..0139e61 100644
--- a/NEWS
+++ b/NEWS
@@ -1,13 +1,13 @@
 v1.8.2
 perf: *much* faster dictionary compression on small files, by @felixhandte
+perf: improved decompression speed and binary size, by Alexey Tourbin (@svpv)
 perf: slightly faster HC compression and decompression speed
 perf: very small compression ratio improvement
-perf: improved decompression binary size and speed, by Alexey Tourbin (@svpv)
 fix : compression compatible with low memory addresses (< 0xFFFF)
 fix : decompression segfault when provided with NULL input, by @terrelln
 cli : new command --favor-decSpeed
 cli : benchmark mode more accurate for small inputs
-fullbench : can measure _destSize() variants, by @felixhandte
+fullbench : can bench _destSize() variants, by @felixhandte
 doc : clarified block format parsing restrictions, by Alexey Tourbin (@svpv)
 
 v1.8.1
-- 
cgit v0.12


From 5406c2e479f5bb2b594a43e74e28719f4640450f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 2 May 2018 23:29:07 -0400
Subject: Only Reset the LZ4 Stream when Init'ing a Streaming Block

---
 lib/lz4.h      | 24 +++++++++++++++++++-----
 lib/lz4frame.c | 19 ++++++++++++++-----
 2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index db6353c..d8238f5 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -358,6 +358,15 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 #ifdef LZ4_STATIC_LINKING_ONLY
 
 /*! LZ4_resetStream_fast() :
+ *  Use this, like LZ4_resetStream(), to prepare a context for a new chain of
+ *  calls to a streaming API (e.g., LZ4_compress_fast_continue()).
+ *
+ *  Note:
+ *  Using this in advance of a non- streaming-compression function is redundant,
+ *  and potentially bad for performance, since they all perform their own custom
+ *  reset internally.
+ *
+ *  Differences from LZ4_resetStream():
  *  When an LZ4_stream_t is known to be in a internally coherent state,
  *  it can often be prepared for a new compression with almost no work, only
  *  sometimes falling back to the full, expensive reset that is always required
@@ -367,13 +376,17 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
  *  LZ4_streams are guaranteed to be in a valid state when:
  *  - returned from LZ4_createStream()
  *  - reset by LZ4_resetStream()
- *  - memset(stream, 0, sizeof(LZ4_stream_t))
+ *  - memset(stream, 0, sizeof(LZ4_stream_t)), though this is discouraged
  *  - the stream was in a valid state and was reset by LZ4_resetStream_fast()
  *  - the stream was in a valid state and was then used in any compression call
  *    that returned success
  *  - the stream was in an indeterminate state and was used in a compression
- *    call that fully reset the state (LZ4_compress_fast_extState()) and that
- *    returned success
+ *    call that fully reset the state (e.g., LZ4_compress_fast_extState()) and
+ *    that returned success
+ *
+ *  When a stream isn't known to be in a valid state, it is not safe to pass to
+ *  any fastReset or streaming function. It must first be cleansed by the full
+ *  LZ4_resetStream().
  */
 LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
 
@@ -384,8 +397,9 @@ LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
  *  to call if the state buffer is known to be correctly initialized already
  *  (see above comment on LZ4_resetStream_fast() for a definition of "correctly
  *  initialized"). From a high level, the difference is that this function
- *  initializes the provided state with a call to LZ4_resetStream_fast() while
- *  LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
+ *  initializes the provided state with a call to something like
+ *  LZ4_resetStream_fast() while LZ4_compress_fast_extState() starts with a
+ *  call to LZ4_resetStream().
  */
 LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 6cf2a06..91e5a43 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -537,9 +537,18 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
  */
 static void LZ4F_initStream(void* ctx,
                             const LZ4F_CDict* cdict,
-                            int level) {
+                            int level,
+                            LZ4F_blockMode_t blockMode) {
     if (level < LZ4HC_CLEVEL_MIN) {
-        LZ4_resetStream_fast((LZ4_stream_t *)ctx);
+        if (cdict != NULL || blockMode == LZ4F_blockLinked) {
+            /* In these cases, we will call LZ4_compress_fast_continue(),
+             * which needs an already reset context. Otherwise, we'll call a
+             * one-shot API. The non-continued APIs internally perform their own
+             * resets at the beginning of their calls, where they know what
+             * tableType they need the context to be in. So in that case this
+             * would be misguided / wasted work. */
+            LZ4_resetStream_fast((LZ4_stream_t*)ctx);
+        }
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
         LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
@@ -617,7 +626,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->cdict = cdict;
     if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
         /* frame init only for blockLinked : blockIndependent will be init at each block */
-        LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
+        LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel, LZ4F_blockLinked);
     }
     if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
           LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
@@ -715,7 +724,7 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
-    LZ4F_initStream(ctx, cdict, level);
+    LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
     if (cdict) {
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
@@ -732,7 +741,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
 
 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
-    LZ4F_initStream(ctx, cdict, level);
+    LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
     if (cdict) {
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }
-- 
cgit v0.12


From 2e2c9f6ff353e9f1a4d23274eb4e5b7a5f7d654d Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Thu, 3 May 2018 07:56:33 -0700
Subject: fix comments / indentation

as requested by @terrelln
---
 lib/lz4.c | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 71fe8f3..3860c51 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1427,10 +1427,9 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
          * those 18 bytes earlier, upon entering the shortcut (in other words,
          * there is a combined check for both stages).
          */
-        if ((endOnInput ? length != RUN_MASK : length <= 8) &&
+        if ( (endOnInput ? length != RUN_MASK : length <= 8)
             /* strictly "less than" on input, to re-enter the loop with at least one byte */
-            likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)))
-        {
+          && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) {
             /* Copy the literals */
             memcpy(op, ip, endOnInput ? 16 : 8);
             op += length; ip += length;
@@ -1442,9 +1441,9 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             match = op - offset;
 
             /* Do not deal with overlapping matches. */
-            if ((length != 15) && (offset >= 8) &&
-                (dict==withPrefix64k || match >= lowPrefix))
-            {
+            if ( (length != ML_MASK)
+              && (offset >= 8)
+              && (dict==withPrefix64k || match >= lowPrefix) ) {
                 /* Copy the match. */
                 memcpy(op + 0, match + 0, 8);
                 memcpy(op + 8, match + 8, 8);
@@ -1709,13 +1708,12 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
  *  provides the minimum size of this ring buffer
  *  to be compatible with any source respecting maxBlockSize condition.
  *  Note : in a ring buffer scenario,
- *  blocks are presumed decompressed next to each other
- *  up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
- *  at which stage it resumes from beginning of ring buffer.
+ *  blocks are presumed decompressed next to each other.
+ *  When not enough space remains for next block (remainingSize < maxBlockSize),
+ *  decoding resumes from beginning of ring buffer.
  * @return : minimum ring buffer size,
  *           or 0 if there is an error (invalid maxBlockSize).
  */
-
 int LZ4_decoderRingBufferSize(int maxBlockSize)
 {
     if (maxBlockSize < 0) return 0;
-- 
cgit v0.12


From dc4270710739296602235caa5cfd065feafd87b7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 3 May 2018 15:38:32 -0700
Subject: created LZ4HC_FindLongestMatch()

simplified match finder
only searching forward and within current buffer,
for easier testing of optimizations.
---
 lib/lz4hc.c | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 88 insertions(+), 16 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 0859ea6..132673e 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1083,6 +1083,80 @@ LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
 }
 
 
+int
+LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
+                        const BYTE* const ip,
+                        const BYTE* const iHighLimit,
+                        int longest,
+                        const BYTE** matchpos,
+                        const int maxNbAttempts)
+{
+    U16* const chainTable = hc4->chainTable;
+    U32* const HashTable = hc4->hashTable;
+    const BYTE* const base = hc4->base;
+    const U32 ipIndex = (U32)(ip - base);
+    const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
+    int nbAttempts = maxNbAttempts;
+    U32 const pattern = LZ4_read32(ip);
+    U32 matchIndex;
+
+    DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch");
+    /* First Match */
+    LZ4HC_Insert(hc4, ip);
+    matchIndex = HashTable[LZ4HC_hashPtr(ip)];
+    DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)",
+                matchIndex, lowestMatchIndex);
+
+    /* find first match */
+    while ( (longest <= MINMATCH)
+      && (matchIndex>=lowestMatchIndex)
+      && (nbAttempts) ) {
+        const BYTE* const matchPtr = base + matchIndex;
+        assert(matchIndex < ipIndex);
+        assert(matchIndex >= dictLimit);
+        assert(matchPtr >= lowPrefixPtr);
+        assert(matchPtr < ip);
+        assert(longest >= 1);
+        nbAttempts--;
+        if (LZ4_read32(matchPtr) == pattern) {
+            int const mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+            if (mlt > longest) {
+                longest = mlt;
+                *matchpos = matchPtr;
+                break;
+        }   }
+
+        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
+            matchIndex -= nextOffset;
+        }
+    }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
+
+    assert(longest > MINMATCH);
+    while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
+        const BYTE* const matchPtr = base + matchIndex;
+        assert(matchIndex < ipIndex);
+        assert(matchIndex >= dictLimit);
+        assert(matchPtr >= lowPrefixPtr);
+        assert(matchPtr < ip);
+        assert(longest >= 1);
+        nbAttempts--;
+        if (LZ4_read16(ip + longest - 1) == LZ4_read16(matchPtr + longest - 1)) {
+            if (LZ4_read32(matchPtr) == pattern) {
+                int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+                if (mlt > longest) {
+                    longest = mlt;
+                    *matchpos = matchPtr;
+        }   }   }
+
+        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
+            matchIndex -= nextOffset;
+        }
+    }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
+
+    return longest;
+}
+
+
 typedef struct {
     int off;
     int len;
@@ -1100,9 +1174,8 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
-                                ip, ip, iHighLimit, minLen, &matchPtr, &ip,
-                                nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
+    //int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
+    int matchLength = LZ4HC_FindLongestMatch(ctx, ip, iHighLimit, minLen, &matchPtr, nbSearches); (void)dict;
     if (matchLength <= minLen) return match;
     if (favorDecSpeed) {
         if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */
@@ -1112,19 +1185,18 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
     return match;
 }
 
-static int LZ4HC_compress_optimal (
-    LZ4HC_CCtx_internal* ctx,
-    const char* const source,
-    char* dst,
-    int* srcSizePtr,
-    int dstCapacity,
-    int const nbSearches,
-    size_t sufficient_len,
-    const limitedOutput_directive limit,
-    int const fullUpdate,
-    const dictCtx_directive dict,
-    const HCfavor_e favorDecSpeed
-    )
+
+static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
+                                    const char* const source,
+                                    char* dst,
+                                    int* srcSizePtr,
+                                    int dstCapacity,
+                                    int const nbSearches,
+                                    size_t sufficient_len,
+                                    const limitedOutput_directive limit,
+                                    int const fullUpdate,
+                                    const dictCtx_directive dict,
+                                    const HCfavor_e favorDecSpeed)
 {
 #define TRAILING_LITERALS 3
     LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS];   /* ~64 KB, which is a bit large for stack... */
-- 
cgit v0.12


From e00ba49cdeb6a6d7b219565286e45d23801429d6 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 3 May 2018 15:40:01 -0700
Subject: updated API documentation

---
 doc/lz4_manual.html | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index e079db2..e5044fe 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -278,7 +278,16 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
 <BR></pre>
 
 <pre><b>void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
-</b><p>  When an LZ4_stream_t is known to be in a internally coherent state,
+</b><p>  Use this, like LZ4_resetStream(), to prepare a context for a new chain of
+  calls to a streaming API (e.g., LZ4_compress_fast_continue()).
+
+  Note:
+  Using this in advance of a non- streaming-compression function is redundant,
+  and potentially bad for performance, since they all perform their own custom
+  reset internally.
+
+  Differences from LZ4_resetStream():
+  When an LZ4_stream_t is known to be in a internally coherent state,
   it can often be prepared for a new compression with almost no work, only
   sometimes falling back to the full, expensive reset that is always required
   when the stream is in an indeterminate state (i.e., the reset performed by
@@ -287,13 +296,17 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
   LZ4_streams are guaranteed to be in a valid state when:
   - returned from LZ4_createStream()
   - reset by LZ4_resetStream()
-  - memset(stream, 0, sizeof(LZ4_stream_t))
+  - memset(stream, 0, sizeof(LZ4_stream_t)), though this is discouraged
   - the stream was in a valid state and was reset by LZ4_resetStream_fast()
   - the stream was in a valid state and was then used in any compression call
     that returned success
   - the stream was in an indeterminate state and was used in a compression
-    call that fully reset the state (LZ4_compress_fast_extState()) and that
-    returned success
+    call that fully reset the state (e.g., LZ4_compress_fast_extState()) and
+    that returned success
+
+  When a stream isn't known to be in a valid state, it is not safe to pass to
+  any fastReset or streaming function. It must first be cleansed by the full
+  LZ4_resetStream().
  
 </p></pre><BR>
 
@@ -304,8 +317,9 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
   to call if the state buffer is known to be correctly initialized already
   (see above comment on LZ4_resetStream_fast() for a definition of "correctly
   initialized"). From a high level, the difference is that this function
-  initializes the provided state with a call to LZ4_resetStream_fast() while
-  LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
+  initializes the provided state with a call to something like
+  LZ4_resetStream_fast() while LZ4_compress_fast_extState() starts with a
+  call to LZ4_resetStream().
  
 </p></pre><BR>
 
-- 
cgit v0.12


From d358e33faa87e0293f27f272a8579de73f4ba938 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 3 May 2018 16:01:24 -0700
Subject: Added CDict speed graph to be used for release statement

---
 doc/images/usingCDict_1_8_2.png | Bin 0 -> 81858 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 doc/images/usingCDict_1_8_2.png

diff --git a/doc/images/usingCDict_1_8_2.png b/doc/images/usingCDict_1_8_2.png
new file mode 100644
index 0000000..9434198
Binary files /dev/null and b/doc/images/usingCDict_1_8_2.png differ
-- 
cgit v0.12


From 434ace7244b063c418b8d9f5c29330411a1df237 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 3 May 2018 16:31:41 -0700
Subject: implemented search accelerator

greatly improves speed compared to non-accelerated,
especially for slower files.

On my laptop, -b12 :
```
calgary.tar :  4.3 MB/s =>  9.0 MB/s
enwik7      : 10.2 MB/s => 13.3 MB/s
silesia.tar :  4.0 MB/s =>  8.7 MB/s
```

Note : this is the simplified version,
without handling dictionaries, external buffer, nor pattern analyzer.
Current `dev` branch on these samples gives :
```
calgary.tar :  4.2 MB/s
enwik7      :  9.7 MB/s
silesia.tar :  3.5 MB/s
```

interestingly, it's slower,
presumably due to handling of dictionaries.
---
 lib/lz4hc.c | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 132673e..6c50db5 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1099,6 +1099,7 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
     int nbAttempts = maxNbAttempts;
     U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
+    int fromMStart = 0;
 
     DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch");
     /* First Match */
@@ -1132,8 +1133,8 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
     }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
 
     assert(longest > MINMATCH);
-    while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
-        const BYTE* const matchPtr = base + matchIndex;
+    while ((matchIndex-fromMStart>=lowestMatchIndex) && (nbAttempts)) {
+        const BYTE* const matchPtr = base + matchIndex - fromMStart;
         assert(matchIndex < ipIndex);
         assert(matchIndex >= dictLimit);
         assert(matchPtr >= lowPrefixPtr);
@@ -1146,6 +1147,21 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
                 if (mlt > longest) {
                     longest = mlt;
                     *matchpos = matchPtr;
+                    matchIndex -= fromMStart;   /* beginning of match */
+                    if (1 && matchIndex + longest <= ipIndex) {
+                        U32 distanceToNextMatch = 1;
+                        int pos;
+                        for (pos = 0; pos <= longest - MINMATCH; pos++) {
+                            U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
+                            if (candidateDist > distanceToNextMatch) {
+                                distanceToNextMatch = candidateDist;
+                                fromMStart = pos;
+                            }
+                        }
+                        if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
+                        matchIndex -= distanceToNextMatch - fromMStart;
+                        continue;
+                    }
         }   }   }
 
         {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
-- 
cgit v0.12


From a7cc0b590a634e3c2612976c1dc83e74f8448d43 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 4 May 2018 13:35:10 -0700
Subject: Fix make install

* Uninstall didn't remove the pkg-config correctly.
* Fix `mandir`
* Allow overriding either upper- or lower-case location variables, but
  always use the lower case variables.
* Add test case that ensures overriding both upper- and lower-case
  variables is the same, and that the directory is empty after uninstall.
---
 lib/Makefile          | 70 ++++++++++++++++++++++++++-------------------------
 programs/Makefile     | 62 ++++++++++++++++++++++-----------------------
 tests/Makefile        |  5 +++-
 tests/test_install.sh | 21 ++++++++++++++++
 4 files changed, 91 insertions(+), 67 deletions(-)
 create mode 100755 tests/test_install.sh

diff --git a/lib/Makefile b/lib/Makefile
index d63de18..abb6c07 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -62,7 +62,7 @@ ifeq ($(shell uname), Darwin)
 	SHARED_EXT = dylib
 	SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
 	SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
-	SONAME_FLAGS = -install_name $(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
+	SONAME_FLAGS = -install_name $(libdir)/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
 else
 	SONAME_FLAGS = -Wl,-soname=liblz4.$(SHARED_EXT).$(LIBVER_MAJOR)
 	SHARED_EXT = so
@@ -132,20 +132,22 @@ listL120:  # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (n
 DESTDIR     ?=
 # directory variables : GNU conventions prefer lowercase
 # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
-# support both lower and uppercase (BSD), use uppercase in script
-prefix      ?= /usr/local
-PREFIX      ?= $(prefix)
-exec_prefix ?= $(PREFIX)
-libdir      ?= $(exec_prefix)/lib
-LIBDIR      ?= $(libdir)
-includedir  ?= $(PREFIX)/include
-INCLUDEDIR  ?= $(includedir)
+# support both lower and uppercase (BSD), use lower in script
+PREFIX      ?= /usr/local
+prefix      ?= $(PREFIX)
+EXEC_PREFIX ?= $(prefix)
+exec_prefix ?= $(EXEC_PREFIX)
+LIBDIR      ?= $(exec_prefix)/lib
+libdir      ?= $(LIBDIR)
+INCLUDEDIR  ?= $(prefix)/include
+includedir  ?= $(INCLUDEDIR)
 
 ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
-PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
+PKGCONFIGDIR ?= $(prefix)/libdata/pkgconfig
 else
-PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
+PKGCONFIGDIR ?= $(libdir)/pkgconfig
 endif
+pkgconfigdir ?= $(PKGCONFIGDIR)
 
 ifneq (,$(filter $(shell uname),SunOS))
 INSTALL ?= ginstall
@@ -158,41 +160,41 @@ INSTALL_DATA    ?= $(INSTALL) -m 644
 
 liblz4.pc: liblz4.pc.in Makefile
 	@echo creating pkgconfig
-	$(Q)sed -e 's|@PREFIX@|$(PREFIX)|' \
-         -e 's|@LIBDIR@|$(LIBDIR)|' \
-         -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
+	$(Q)sed -e 's|@PREFIX@|$(prefix)|' \
+         -e 's|@LIBDIR@|$(libdir)|' \
+         -e 's|@INCLUDEDIR@|$(includedir)|' \
          -e 's|@VERSION@|$(LIBVER)|' \
           $< >$@
 
 install: lib liblz4.pc
-	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ $(DESTDIR)$(LIBDIR)/
-	$(Q)$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/
+	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/
+	$(Q)$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(pkgconfigdir)/
 	@echo Installing libraries
 ifeq ($(BUILD_STATIC),yes)
-	$(Q)$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
-	$(Q)$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
+	$(Q)$(INSTALL_DATA) liblz4.a $(DESTDIR)$(libdir)/liblz4.a
+	$(Q)$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(includedir)/lz4frame_static.h
 endif
 ifeq ($(BUILD_SHARED),yes)
-	$(Q)$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
-	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
-	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
+	$(Q)$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)
+	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
+	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
 endif
-	@echo Installing headers in $(INCLUDEDIR)
-	$(Q)$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
-	$(Q)$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
-	$(Q)$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
+	@echo Installing headers in $(includedir)
+	$(Q)$(INSTALL_DATA) lz4.h $(DESTDIR)$(includedir)/lz4.h
+	$(Q)$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(includedir)/lz4hc.h
+	$(Q)$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(includedir)/lz4frame.h
 	@echo lz4 libraries installed
 
 uninstall:
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
+	$(Q)$(RM) $(DESTDIR)$(pkgconfigdir)/liblz4.pc
+	$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
+	$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
+	$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_VER)
+	$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.a
+	$(Q)$(RM) $(DESTDIR)$(includedir)/lz4.h
+	$(Q)$(RM) $(DESTDIR)$(includedir)/lz4hc.h
+	$(Q)$(RM) $(DESTDIR)$(includedir)/lz4frame.h
+	$(Q)$(RM) $(DESTDIR)$(includedir)/lz4frame_static.h
 	@echo lz4 libraries successfully uninstalled
 
 endif
diff --git a/programs/Makefile b/programs/Makefile
index a51bd4b..72bdcaa 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -120,21 +120,19 @@ lz4cat: lz4
 DESTDIR     ?=
 # directory variables : GNU conventions prefer lowercase
 # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
-# support both lower and uppercase (BSD), use uppercase in script
-prefix      ?= /usr/local
-PREFIX      ?= $(prefix)
-exec_prefix ?= $(PREFIX)
-bindir      ?= $(exec_prefix)/bin
-BINDIR      ?= $(bindir)
-datarootdir ?= $(PREFIX)/share
-mandir      ?= $(datarootdir)/man
-man1dir     ?= $(mandir)/man1
-
-ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS))
-MANDIR  ?= $(PREFIX)/man/man1
-else
-MANDIR  ?= $(man1dir)
-endif
+# support both lower and uppercase (BSD), use lowercase in script
+PREFIX      ?= /usr/local
+prefix      ?= $(PREFIX)
+EXEC_PREFIX ?= $(prefix)
+exec_prefix ?= $(EXEC_PREFIX)
+BINDIR      ?= $(exec_prefix)/bin
+bindir      ?= $(BINDIR)
+DATAROOTDIR ?= $(prefix)/share
+datarootdir ?= $(DATAROOTDIR)
+MANDIR      ?= $(datarootdir)/man
+mandir      ?= $(MANDIR)
+MAN1DIR     ?= $(mandir)/man1
+man1dir     ?= $(MAN1DIR)
 
 ifneq (,$(filter $(shell uname),SunOS))
 INSTALL ?= ginstall
@@ -148,27 +146,27 @@ INSTALL_DATA    ?= $(INSTALL) -m 644
 
 install: lz4
 	@echo Installing binaries
-	@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
-	@$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4
-	@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4c
-	@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat
-	@ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4
+	@$(INSTALL) -d -m 755 $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/
+	@$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(bindir)/lz4
+	@ln -sf lz4 $(DESTDIR)$(bindir)/lz4c
+	@ln -sf lz4 $(DESTDIR)$(bindir)/lz4cat
+	@ln -sf lz4 $(DESTDIR)$(bindir)/unlz4
 	@echo Installing man pages
-	@$(INSTALL_DATA) lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
-	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1
-	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1
-	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1
+	@$(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1
+	@ln -sf lz4.1 $(DESTDIR)$(man1dir)/lz4c.1
+	@ln -sf lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1
+	@ln -sf lz4.1 $(DESTDIR)$(man1dir)/unlz4.1
 	@echo lz4 installation completed
 
 uninstall:
-	@$(RM) $(DESTDIR)$(BINDIR)/lz4cat
-	@$(RM) $(DESTDIR)$(BINDIR)/unlz4
-	@$(RM) $(DESTDIR)$(BINDIR)/lz4
-	@$(RM) $(DESTDIR)$(BINDIR)/lz4c
-	@$(RM) $(DESTDIR)$(MANDIR)/lz4.1
-	@$(RM) $(DESTDIR)$(MANDIR)/lz4c.1
-	@$(RM) $(DESTDIR)$(MANDIR)/lz4cat.1
-	@$(RM) $(DESTDIR)$(MANDIR)/unlz4.1
+	@$(RM) $(DESTDIR)$(bindir)/lz4cat
+	@$(RM) $(DESTDIR)$(bindir)/unlz4
+	@$(RM) $(DESTDIR)$(bindir)/lz4
+	@$(RM) $(DESTDIR)$(bindir)/lz4c
+	@$(RM) $(DESTDIR)$(man1dir)/lz4.1
+	@$(RM) $(DESTDIR)$(man1dir)/lz4c.1
+	@$(RM) $(DESTDIR)$(man1dir)/lz4cat.1
+	@$(RM) $(DESTDIR)$(man1dir)/unlz4.1
 	@echo lz4 programs successfully uninstalled
 
 endif
diff --git a/tests/Makefile b/tests/Makefile
index d4847b1..196426e 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -141,11 +141,14 @@ endif
 DD:=dd
 
 
-test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer
+test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install
 
 test32: CFLAGS+=-m32
 test32: test
 
+test-install:
+	lz4_root=.. ./test_install.sh
+
 test-lz4-sparse: lz4 datagen
 	@echo "\n ---- test sparse file support ----"
 	./datagen -g5M  -P100 > tmplsdg5M
diff --git a/tests/test_install.sh b/tests/test_install.sh
new file mode 100755
index 0000000..2e1a5a5
--- /dev/null
+++ b/tests/test_install.sh
@@ -0,0 +1,21 @@
+#/usr/bin/env sh
+set -e
+
+make="make -C $lz4_root"
+$make CFLAGS=-O0 > /dev/null
+for cmd in install uninstall; do
+  for upper in DUMMY PREFIX EXEC_PREFIX LIBDIR INCLUDEDIR PKGCONFIGDIR BINDIR MANDIR MAN1DIR ; do
+    lower=$(echo $upper | tr '[:upper:]' '[:lower:]')
+    tmp_lower="$(pwd)/tmp-lower-$lower/"
+    tmp_upper="$(pwd)/tmp-upper-$lower/"
+    echo $make $cmd DESTDIR="$tmp_upper" $upper="test"
+    $make $cmd DESTDIR="$tmp_upper" $upper="test" >/dev/null
+    echo $make $cmd DESTDIR="$tmp_lower" $lower="test"
+    $make $cmd DESTDIR="$tmp_lower" $lower="test" >/dev/null
+    command diff -r "$tmp_lower" "$tmp_upper" && echo "SAME!" || false
+    if [ "x$cmd" = "xuninstall" ]; then
+      test -z "$(find "$tmp_lower" -type f)" && echo "EMPTY!" || false
+      rm -rf "$tmp_upper" "$tmp_lower"
+    fi
+  done
+done
-- 
cgit v0.12


From 3e7fa1b14d9a5fc7f10c36231ef5cd2ba3ac7350 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 4 May 2018 14:33:59 -0700
Subject: Attempt to fix travis

---
 tests/Makefile        | 5 ++++-
 tests/test_install.sh | 1 -
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 196426e..d238561 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -71,6 +71,9 @@ all32: all
 lz4:
 	$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)"
 
+lib liblz4.pc:
+	$(MAKE) -C $(LZ4DIR) $@ CFLAGS="$(CFLAGS)"
+
 lz4c unlz4 lz4cat: lz4
 	ln -sf $(LZ4) $(PRGDIR)/$@
 
@@ -146,7 +149,7 @@ test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install
 test32: CFLAGS+=-m32
 test32: test
 
-test-install:
+test-install: lz4 lib liblz4.pc
 	lz4_root=.. ./test_install.sh
 
 test-lz4-sparse: lz4 datagen
diff --git a/tests/test_install.sh b/tests/test_install.sh
index 2e1a5a5..f9de402 100755
--- a/tests/test_install.sh
+++ b/tests/test_install.sh
@@ -2,7 +2,6 @@
 set -e
 
 make="make -C $lz4_root"
-$make CFLAGS=-O0 > /dev/null
 for cmd in install uninstall; do
   for upper in DUMMY PREFIX EXEC_PREFIX LIBDIR INCLUDEDIR PKGCONFIGDIR BINDIR MANDIR MAN1DIR ; do
     lower=$(echo $upper | tr '[:upper:]' '[:lower:]')
-- 
cgit v0.12


From 9699ba5ddfa8485bb4ad36cc9ec78cbed542f28b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 4 May 2018 19:13:33 -0700
Subject: integrated chain swapper into HC match finder

slower than expected
Pattern analyzer and Chain Swapper
work slower when both activated.
Reasons unclear.
---
 lib/lz4hc.c | 121 ++++++++++++++++++++++++++++++++++++++----------------------
 1 file changed, 76 insertions(+), 45 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 6c50db5..c58cd9d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -227,6 +227,7 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE* const dictBase = hc4->dictBase;
     int const lookBackLength = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
+    int matchChainPos = 0;
     U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
     U32 dictMatchIndex;
@@ -241,31 +242,30 @@ LZ4HC_InsertAndGetWiderMatch (
                 matchIndex, lowestMatchIndex);
 
     while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
-        DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
+        int mlt=0;
         nbAttempts--;
         assert(matchIndex < ipIndex);
         if (favorDecSpeed && (ipIndex - matchIndex < 8)) {
             /* do nothing */
-        } else if (matchIndex >= dictLimit) {
+        } else if (matchIndex >= dictLimit) {   /* within current Prefix */
             const BYTE* const matchPtr = base + matchIndex;
             assert(matchPtr >= lowPrefixPtr);
             assert(matchPtr < ip);
             assert(longest >= 1);
             if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
-                    int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
                     int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
+                    mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
                     mlt -= back;
                     if (mlt > longest) {
                         longest = mlt;
-                        *matchpos = matchPtr+back;
-                        *startpos = ip+back;
+                        *matchpos = matchPtr + back;
+                        *startpos = ip + back;
             }   }   }
-        } else {   /* matchIndex < dictLimit */
+        } else {   /* lowestMatchIndex <= matchIndex < dictLimit */
             const BYTE* const matchPtr = dictBase + matchIndex;
             if (LZ4_read32(matchPtr) == pattern) {
                 const BYTE* const dictStart = dictBase + hc4->lowLimit;
-                int mlt;
                 int back = 0;
                 const BYTE* vLimit = ip + (dictLimit - matchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
@@ -280,9 +280,10 @@ LZ4HC_InsertAndGetWiderMatch (
                     *startpos = ip + back;
         }   }   }
 
-        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
-            matchIndex -= nextOffset;
-            if (patternAnalysis && nextOffset==1) {
+        {   U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex);
+            if (1 && patternAnalysis && distNextMatch==1 && matchChainPos==0) {
+                U32 const matchCandidateIdx = matchIndex-1;
+                //static U64 g_nbPatternAnalysis = 0; g_nbPatternAnalysis++; if (g_nbPatternAnalysis % (((g_nbPatternAnalysis/1000000)+1)*100)==0) DEBUGLOG(2, "g_nbPatternAnalysis=%llu ", g_nbPatternAnalysis);
                 /* may be a repeated pattern */
                 if (repeat == rep_untested) {
                     if ( ((pattern & 0xFFFF) == (pattern >> 16))
@@ -293,21 +294,48 @@ LZ4HC_InsertAndGetWiderMatch (
                         repeat = rep_not;
                 }   }
                 if ( (repeat == rep_confirmed)
-                  && (matchIndex >= dictLimit) ) {   /* same segment only */
-                    const BYTE* const matchPtr = base + matchIndex;
+                  && (matchCandidateIdx >= dictLimit) ) {   /* same segment only */
+                    const BYTE* const matchPtr = base + matchCandidateIdx;
                     if (LZ4_read32(matchPtr) == pattern) {  /* good candidate */
                         size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
                         const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
                         size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern);
                         size_t const currentSegmentLength = backLength + forwardPatternLength;
+                        //static U64 g_nbPatternAnalysis_confirmed = 0; g_nbPatternAnalysis_confirmed++; if (g_nbPatternAnalysis_confirmed%(((g_nbPatternAnalysis_confirmed/100000)+1)*100)==0) DEBUGLOG(2, "g_nbPatternAnalysis_confirmed=%llu ", g_nbPatternAnalysis_confirmed);
 
                         if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
                           && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
-                            matchIndex += (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
+                            matchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
                         } else {
-                            matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
+                            matchIndex = matchCandidateIdx - (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
-        }   }   }   }
+                        matchChainPos = 0;
+                        continue;
+                }   }
+        }   }
+
+        if (mlt == longest) {  /* better match => select a better chain */
+            assert(longest >= MINMATCH);
+            if (1 && matchIndex + longest <= ipIndex) {
+                U32 distanceToNextMatch = 1;
+                int pos;
+                //static U64 g_nbChainSwapper = 0; g_nbChainSwapper++; if (g_nbChainSwapper%(((g_nbChainSwapper/100000)+1)*100)==0) DEBUGLOG(2, "g_nbChainSwapper=%llu ", g_nbChainSwapper);
+                for (pos = matchChainPos; pos <= longest - MINMATCH; pos++) {
+                    U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
+                    if (candidateDist > distanceToNextMatch) {
+                        distanceToNextMatch = candidateDist;
+                        matchChainPos = pos;
+                    }
+                }
+                if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
+                matchIndex -= distanceToNextMatch;
+                continue;
+            }
+        }
+
+        /* follow current chain */
+        matchIndex -= DELTANEXTU16(chainTable, matchIndex+matchChainPos);
+
     }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
 
     if (dict == usingDictCtx && nbAttempts && ipIndex - lowestMatchIndex < MAX_DISTANCE) {
@@ -339,6 +367,7 @@ LZ4HC_InsertAndGetWiderMatch (
             }
         }
     }
+
     return longest;
 }
 
@@ -1099,7 +1128,7 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
     int nbAttempts = maxNbAttempts;
     U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
-    int fromMStart = 0;
+    int matchChainPos = 0;
 
     DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch");
     /* First Match */
@@ -1109,13 +1138,11 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
                 matchIndex, lowestMatchIndex);
 
     /* find first match */
-    while ( (longest <= MINMATCH)
+    while ( 1 && (longest < MINMATCH)
       && (matchIndex>=lowestMatchIndex)
       && (nbAttempts) ) {
         const BYTE* const matchPtr = base + matchIndex;
         assert(matchIndex < ipIndex);
-        assert(matchIndex >= dictLimit);
-        assert(matchPtr >= lowPrefixPtr);
         assert(matchPtr < ip);
         assert(longest >= 1);
         nbAttempts--;
@@ -1124,7 +1151,6 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
             if (mlt > longest) {
                 longest = mlt;
                 *matchpos = matchPtr;
-                break;
         }   }
 
         {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
@@ -1132,39 +1158,44 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
         }
     }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
 
-    assert(longest > MINMATCH);
-    while ((matchIndex-fromMStart>=lowestMatchIndex) && (nbAttempts)) {
-        const BYTE* const matchPtr = base + matchIndex - fromMStart;
+    /* search longer matches */
+    while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
+        const BYTE* const matchPtr = base + matchIndex;
+        int mlt = 0;
         assert(matchIndex < ipIndex);
-        assert(matchIndex >= dictLimit);
-        assert(matchPtr >= lowPrefixPtr);
         assert(matchPtr < ip);
-        assert(longest >= 1);
+        assert(longest >= MINMATCH);
         nbAttempts--;
         if (LZ4_read16(ip + longest - 1) == LZ4_read16(matchPtr + longest - 1)) {
             if (LZ4_read32(matchPtr) == pattern) {
-                int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+                mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
                 if (mlt > longest) {
                     longest = mlt;
                     *matchpos = matchPtr;
-                    matchIndex -= fromMStart;   /* beginning of match */
-                    if (1 && matchIndex + longest <= ipIndex) {
-                        U32 distanceToNextMatch = 1;
-                        int pos;
-                        for (pos = 0; pos <= longest - MINMATCH; pos++) {
-                            U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
-                            if (candidateDist > distanceToNextMatch) {
-                                distanceToNextMatch = candidateDist;
-                                fromMStart = pos;
-                            }
-                        }
-                        if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
-                        matchIndex -= distanceToNextMatch - fromMStart;
-                        continue;
+                }
+            }
+        }
+
+        /* search more promising chain */
+        if (mlt == longest) {
+            if (1 && matchIndex + longest <= ipIndex) {
+                U32 distanceToNextMatch = 1;
+                int pos;
+                for (pos = matchChainPos; pos <= longest - MINMATCH; pos++) {
+                    U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
+                    if (candidateDist > distanceToNextMatch) {
+                        distanceToNextMatch = candidateDist;
+                        matchChainPos = pos;
                     }
-        }   }   }
+                }
+                if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
+                matchIndex -= distanceToNextMatch;
+                continue;
+            }
+        }
 
-        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
+        /* continue current chain */
+        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex+matchChainPos);
             matchIndex -= nextOffset;
         }
     }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
@@ -1190,8 +1221,8 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    //int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
-    int matchLength = LZ4HC_FindLongestMatch(ctx, ip, iHighLimit, minLen, &matchPtr, nbSearches); (void)dict;
+    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
+    //int matchLength = LZ4HC_FindLongestMatch(ctx, ip, iHighLimit, minLen, &matchPtr, nbSearches); (void)dict;
     if (matchLength <= minLen) return match;
     if (favorDecSpeed) {
         if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */
-- 
cgit v0.12


From fa89a9e18b93741c65fa84c293d15298169d1ecb Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 13:31:03 -0700
Subject: lz4hc: fixed performance issue

when combining both PA and CS optimizations
---
 lib/lz4hc.c | 134 +++++++++---------------------------------------------------
 1 file changed, 20 insertions(+), 114 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c58cd9d..7e2fcb3 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -280,10 +280,29 @@ LZ4HC_InsertAndGetWiderMatch (
                     *startpos = ip + back;
         }   }   }
 
+        if (mlt == longest) {  /* better match => select a better chain */
+            assert(longest >= MINMATCH);
+            if (1 && matchIndex + longest <= ipIndex) {
+                U32 distanceToNextMatch = 1;
+                int pos;
+                for (pos = 0; pos <= longest - MINMATCH; pos++) {
+                    U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
+                    if (candidateDist > distanceToNextMatch) {
+                        distanceToNextMatch = candidateDist;
+                        matchChainPos = pos;
+                    }
+                }
+                if (distanceToNextMatch > 1) {
+                    if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
+                    matchIndex -= distanceToNextMatch;
+                    continue;
+                }
+            }
+        }
+
         {   U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex);
             if (1 && patternAnalysis && distNextMatch==1 && matchChainPos==0) {
                 U32 const matchCandidateIdx = matchIndex-1;
-                //static U64 g_nbPatternAnalysis = 0; g_nbPatternAnalysis++; if (g_nbPatternAnalysis % (((g_nbPatternAnalysis/1000000)+1)*100)==0) DEBUGLOG(2, "g_nbPatternAnalysis=%llu ", g_nbPatternAnalysis);
                 /* may be a repeated pattern */
                 if (repeat == rep_untested) {
                     if ( ((pattern & 0xFFFF) == (pattern >> 16))
@@ -301,7 +320,6 @@ LZ4HC_InsertAndGetWiderMatch (
                         const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
                         size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern);
                         size_t const currentSegmentLength = backLength + forwardPatternLength;
-                        //static U64 g_nbPatternAnalysis_confirmed = 0; g_nbPatternAnalysis_confirmed++; if (g_nbPatternAnalysis_confirmed%(((g_nbPatternAnalysis_confirmed/100000)+1)*100)==0) DEBUGLOG(2, "g_nbPatternAnalysis_confirmed=%llu ", g_nbPatternAnalysis_confirmed);
 
                         if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
                           && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
@@ -314,25 +332,6 @@ LZ4HC_InsertAndGetWiderMatch (
                 }   }
         }   }
 
-        if (mlt == longest) {  /* better match => select a better chain */
-            assert(longest >= MINMATCH);
-            if (1 && matchIndex + longest <= ipIndex) {
-                U32 distanceToNextMatch = 1;
-                int pos;
-                //static U64 g_nbChainSwapper = 0; g_nbChainSwapper++; if (g_nbChainSwapper%(((g_nbChainSwapper/100000)+1)*100)==0) DEBUGLOG(2, "g_nbChainSwapper=%llu ", g_nbChainSwapper);
-                for (pos = matchChainPos; pos <= longest - MINMATCH; pos++) {
-                    U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
-                    if (candidateDist > distanceToNextMatch) {
-                        distanceToNextMatch = candidateDist;
-                        matchChainPos = pos;
-                    }
-                }
-                if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
-                matchIndex -= distanceToNextMatch;
-                continue;
-            }
-        }
-
         /* follow current chain */
         matchIndex -= DELTANEXTU16(chainTable, matchIndex+matchChainPos);
 
@@ -1112,98 +1111,6 @@ LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
 }
 
 
-int
-LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
-                        const BYTE* const ip,
-                        const BYTE* const iHighLimit,
-                        int longest,
-                        const BYTE** matchpos,
-                        const int maxNbAttempts)
-{
-    U16* const chainTable = hc4->chainTable;
-    U32* const HashTable = hc4->hashTable;
-    const BYTE* const base = hc4->base;
-    const U32 ipIndex = (U32)(ip - base);
-    const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
-    int nbAttempts = maxNbAttempts;
-    U32 const pattern = LZ4_read32(ip);
-    U32 matchIndex;
-    int matchChainPos = 0;
-
-    DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch");
-    /* First Match */
-    LZ4HC_Insert(hc4, ip);
-    matchIndex = HashTable[LZ4HC_hashPtr(ip)];
-    DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)",
-                matchIndex, lowestMatchIndex);
-
-    /* find first match */
-    while ( 1 && (longest < MINMATCH)
-      && (matchIndex>=lowestMatchIndex)
-      && (nbAttempts) ) {
-        const BYTE* const matchPtr = base + matchIndex;
-        assert(matchIndex < ipIndex);
-        assert(matchPtr < ip);
-        assert(longest >= 1);
-        nbAttempts--;
-        if (LZ4_read32(matchPtr) == pattern) {
-            int const mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-            if (mlt > longest) {
-                longest = mlt;
-                *matchpos = matchPtr;
-        }   }
-
-        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
-            matchIndex -= nextOffset;
-        }
-    }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
-
-    /* search longer matches */
-    while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
-        const BYTE* const matchPtr = base + matchIndex;
-        int mlt = 0;
-        assert(matchIndex < ipIndex);
-        assert(matchPtr < ip);
-        assert(longest >= MINMATCH);
-        nbAttempts--;
-        if (LZ4_read16(ip + longest - 1) == LZ4_read16(matchPtr + longest - 1)) {
-            if (LZ4_read32(matchPtr) == pattern) {
-                mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-                if (mlt > longest) {
-                    longest = mlt;
-                    *matchpos = matchPtr;
-                }
-            }
-        }
-
-        /* search more promising chain */
-        if (mlt == longest) {
-            if (1 && matchIndex + longest <= ipIndex) {
-                U32 distanceToNextMatch = 1;
-                int pos;
-                for (pos = matchChainPos; pos <= longest - MINMATCH; pos++) {
-                    U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
-                    if (candidateDist > distanceToNextMatch) {
-                        distanceToNextMatch = candidateDist;
-                        matchChainPos = pos;
-                    }
-                }
-                if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
-                matchIndex -= distanceToNextMatch;
-                continue;
-            }
-        }
-
-        /* continue current chain */
-        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex+matchChainPos);
-            matchIndex -= nextOffset;
-        }
-    }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
-
-    return longest;
-}
-
-
 typedef struct {
     int off;
     int len;
@@ -1222,7 +1129,6 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
     int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
-    //int matchLength = LZ4HC_FindLongestMatch(ctx, ip, iHighLimit, minLen, &matchPtr, nbSearches); (void)dict;
     if (matchLength <= minLen) return match;
     if (favorDecSpeed) {
         if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */
-- 
cgit v0.12


From d097bf93f853ddefdd05321d65cf7b90e3171c14 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 13:46:45 -0700
Subject: fixed SC.opt integration with regular HC parser

Only enabled when searching forward.

note : it slighly improves compression ratio,
but measurably decreases speed.
Trade-off to analyse.
---
 lib/lz4hc.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 7e2fcb3..0f37f42 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -280,9 +280,9 @@ LZ4HC_InsertAndGetWiderMatch (
                     *startpos = ip + back;
         }   }   }
 
-        if (mlt == longest) {  /* better match => select a better chain */
-            assert(longest >= MINMATCH);
-            if (1 && matchIndex + longest <= ipIndex) {
+        if ( lookBackLength==0  /* search forward only */
+          && mlt==longest) {    /* better match => select a better chain */
+            if (matchIndex + longest <= ipIndex) {
                 U32 distanceToNextMatch = 1;
                 int pos;
                 for (pos = 0; pos <= longest - MINMATCH; pos++) {
@@ -301,7 +301,7 @@ LZ4HC_InsertAndGetWiderMatch (
         }
 
         {   U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex);
-            if (1 && patternAnalysis && distNextMatch==1 && matchChainPos==0) {
+            if (patternAnalysis && distNextMatch==1 && matchChainPos==0) {
                 U32 const matchCandidateIdx = matchIndex-1;
                 /* may be a repeated pattern */
                 if (repeat == rep_untested) {
-- 
cgit v0.12


From a4e918d7a6e85f825f4cb1ac1c08a2cf2fc22194 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 14:10:30 -0700
Subject: lz4hc: SC only enabled for opt parser

the trade off is not good for regular HC parser :
compression is a little bit better, but speed cost is too large in comparison.
---
 lib/lz4hc.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 0f37f42..d29a989 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -213,6 +213,7 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE** startpos,
     const int maxNbAttempts,
     const int patternAnalysis,
+    const int chainSwap,
     const dictCtx_directive dict,
     const HCfavor_e favorDecSpeed)
 {
@@ -280,8 +281,8 @@ LZ4HC_InsertAndGetWiderMatch (
                     *startpos = ip + back;
         }   }   }
 
-        if ( lookBackLength==0  /* search forward only */
-          && mlt==longest) {    /* better match => select a better chain */
+        if (chainSwap && mlt==longest) {    /* better match => select a better chain */
+            assert(lookBackLength==0);   /* search forward only */
             if (matchIndex + longest <= ipIndex) {
                 U32 distanceToNextMatch = 1;
                 int pos;
@@ -327,7 +328,6 @@ LZ4HC_InsertAndGetWiderMatch (
                         } else {
                             matchIndex = matchCandidateIdx - (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
-                        matchChainPos = 0;
                         continue;
                 }   }
         }   }
@@ -382,7 +382,7 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
+    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, 0 /* chainSwap */, patternAnalysis, dict, favorCompressionRatio);
 }
 
 
@@ -515,7 +515,7 @@ _Search2:
         if (ip+ml <= mflimit) {
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
-                            maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
+                            maxNbAttempts, 0, patternAnalysis, dict, favorCompressionRatio);
         } else {
             ml2 = ml;
         }
@@ -560,7 +560,7 @@ _Search3:
         if (start2 + ml2 <= mflimit) {
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
-                            maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
+                            maxNbAttempts, 0, patternAnalysis, dict, favorCompressionRatio);
         } else {
             ml3 = ml2;
         }
@@ -1128,7 +1128,7 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
+    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /*chainSwap*/, 1 /*patternAnalysis*/, dict, favorDecSpeed);
     if (matchLength <= minLen) return match;
     if (favorDecSpeed) {
         if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */
-- 
cgit v0.12


From cdb0275b7f7513e13a05c27adea658614c0174d9 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 14:32:57 -0700
Subject: lz4hc: fixed PA / SC parameter order

also :
reserved PA for levels 9+ (instead of 8+).
In most cases, speed is lower, and compression benefit is not worth.
---
 lib/lz4hc.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d29a989..1b3596f 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -382,7 +382,7 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, 0 /* chainSwap */, patternAnalysis, dict, favorCompressionRatio);
+    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio);
 }
 
 
@@ -477,7 +477,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
     )
 {
     const int inputSize = *srcSizePtr;
-    const int patternAnalysis = (maxNbAttempts > 64);   /* levels 8+ */
+    const int patternAnalysis = (maxNbAttempts > 128);   /* levels 9+ */
 
     const BYTE* ip = (const BYTE*) source;
     const BYTE* anchor = ip;
@@ -515,7 +515,7 @@ _Search2:
         if (ip+ml <= mflimit) {
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
-                            maxNbAttempts, 0, patternAnalysis, dict, favorCompressionRatio);
+                            maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio);
         } else {
             ml2 = ml;
         }
@@ -560,7 +560,7 @@ _Search3:
         if (start2 + ml2 <= mflimit) {
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
-                            maxNbAttempts, 0, patternAnalysis, dict, favorCompressionRatio);
+                            maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio);
         } else {
             ml3 = ml2;
         }
@@ -1128,7 +1128,7 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /*chainSwap*/, 1 /*patternAnalysis*/, dict, favorDecSpeed);
+    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed);
     if (matchLength <= minLen) return match;
     if (favorDecSpeed) {
         if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */
-- 
cgit v0.12


From af127334670a5e7b710bbd6adb71aa7c3ef0cd72 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 18:24:11 -0700
Subject: fixed frametest error

The error can be reproduced using following command :
./frametest -v -i100000000 -s1659 -t31096808

It's actually a bug in the stream LZ4 API,
when starting a new stream
and providing a first chunk to complete with size < MINMATCH.
In which case, the chunk becomes a dictionary.
No hash was generated and stored,
but the chunk is accessible as default position 0 points to dictStart,
and position 0 is still within MAX_DISTANCE.
Then, next attempt to read 32-bits from position 0 fails.

The issue would have been mitigated by starting from index 64 KB,
effectively eliminating position 0 as too far away.

The proper fix is to eliminate such "dictionary" as too small.
Which is what this patch does.
---
 lib/lz4.c         | 13 ++++++++++++-
 lib/lz4frame.c    |  1 +
 tests/frametest.c |  8 ++++++--
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 3860c51..9a922b5 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -767,6 +767,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 } else if (dictDirective==usingExtDict) {
                     if (matchIndex < startIndex) {
                         DEBUGLOG(7, "extDict candidate: matchIndex=%5u  <  startIndex=%5u", matchIndex, startIndex);
+                        assert(startIndex - matchIndex >= MINMATCH);
                         match = dictBase + matchIndex;
                         lowLimit = dictionary;
                     } else {
@@ -989,6 +990,7 @@ _last_literals:
     if (outputLimited == fillOutput) {
         *inputConsumed = (int) (((const char*)ip)-source);
     }
+    DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, (int)(((char*)op) - dest));
     return (int)(((char*)op) - dest);
 }
 
@@ -1255,12 +1257,19 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 {
     const tableType_t tableType = byU32;
     LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
-    const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
+    const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
     if (streamPtr->initCheck) return 0;   /* Uninitialized structure detected */
     LZ4_renormDictT(streamPtr, inputSize);   /* avoid index overflow */
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
 
+    /* invalidate tiny dictionaries */
+    if (streamPtr->dictSize < 4) {
+        streamPtr->dictSize = 0;
+        streamPtr->dictionary = (const BYTE*)source;
+        dictEnd = (const BYTE*)source;
+    }
+
     /* Check overlapping input/dictionary space */
     {   const BYTE* sourceEnd = (const BYTE*) source + inputSize;
         if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) {
@@ -1402,6 +1411,8 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
     const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/;
     const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/;
 
+    DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i)", srcSize);
+
     /* Special cases */
     if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT;                      /* targetOutputSize too high => just decode everything */
     if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1;  /* Empty output buffer */
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index aa7889b..4e40816 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -808,6 +808,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
+    DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
diff --git a/tests/frametest.c b/tests/frametest.c
index 7d69ff7..21d883d 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -740,8 +740,9 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un
     size_t p=0;
     const BYTE* b1=(const BYTE*)buff1;
     const BYTE* b2=(const BYTE*)buff2;
+    DISPLAY("locateBuffDiff: looking for error position \n");
     if (nonContiguous) {
-        DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size);
+        DISPLAY("mode %u: non-contiguous output (%zu bytes), cannot search \n", nonContiguous, size);
         return;
     }
     while (p < size && b1[p]==b2[p]) p++;
@@ -838,6 +839,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 size_t const iSize = MIN(sampleMax, (size_t)(iend-ip));
                 size_t const oSize = LZ4F_compressBound(iSize, prefsPtr);
                 cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
+                DISPLAYLEVEL(6, "Sending %zi bytes to compress (stableSrc:%u) \n",
+                                iSize, cOptions.stableSrc);
 
                 result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
                 CHECK(LZ4F_isError(result), "Compression failed (error %i : %s)", (int)result, LZ4F_getErrorName(result));
@@ -882,7 +885,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 dOptions.stableDst = FUZ_rand(&randState) & 1;
                 if (nonContiguousDst==2) dOptions.stableDst = 0;   /* overwrite mode */
                 result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
-                if (LZ4F_getErrorCode(result) == LZ4F_ERROR_contentChecksum_invalid) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst);
+                if (LZ4F_getErrorCode(result) == LZ4F_ERROR_contentChecksum_invalid)
+                    locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst);
                 CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName(result));
                 XXH64_update(&xxh64, op, (U32)oSize);
                 totalOut += oSize;
-- 
cgit v0.12


From d7b6c726edb23fc19757dc84d4546863557bfe20 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 19:59:00 -0700
Subject: small extDict : fixed side-effect

don't fix dictionaries of size 0.
setting dictEnd == source triggers prefix mode,
thus removing possibility to use CDict.
---
 lib/lz4.c         | 9 ++++++---
 lib/lz4frame.c    | 1 +
 tests/frametest.c | 1 +
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 9a922b5..e51a3e0 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -683,13 +683,12 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* Init conditions */
     if (outputLimited == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
+    if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;  /* Size too large (not within 64K limit) */
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
     assert(acceleration >= 1);
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
 
-    if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;  /* Size too large (not within 64K limit) */
-
     /* Update context state */
     if (dictDirective == usingDictCtx) {
         /* Subsequent linked blocks can't use the dictionary. */
@@ -1259,12 +1258,16 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
     LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
     const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
+    DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize);
+
     if (streamPtr->initCheck) return 0;   /* Uninitialized structure detected */
     LZ4_renormDictT(streamPtr, inputSize);   /* avoid index overflow */
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
 
     /* invalidate tiny dictionaries */
-    if (streamPtr->dictSize < 4) {
+    if ( (streamPtr->dictSize-1 < 4)   /* intentional underflow */
+      && (dictEnd != (const BYTE*)source) ) {
+        DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary);
         streamPtr->dictSize = 0;
         streamPtr->dictionary = (const BYTE*)source;
         dictEnd = (const BYTE*)source;
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 4e40816..e1d0b1d 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -472,6 +472,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
     const char* dictStart = (const char*)dictBuffer;
     LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));
+    DEBUGLOG(4, "LZ4F_createCDict");
     if (!cdict) return NULL;
     if (dictSize > 64 KB) {
         dictStart += dictSize - 64 KB;
diff --git a/tests/frametest.c b/tests/frametest.c
index 21d883d..4efeb6f 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -520,6 +520,7 @@ int basicTests(U32 seed, double compressibility)
         LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
         if (cdict == NULL) goto _output_error;
         CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
+        
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
         CHECK_V(cSizeNoDict,
                 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
-- 
cgit v0.12


From 24b9c485db34743f83496038d61598d41c3c497e Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 6 May 2018 16:47:31 -0700
Subject: small PA optimization

which measurably improves speed
on levels 9+
---
 lib/lz4hc.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 1b3596f..059c4b5 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -291,15 +291,12 @@ LZ4HC_InsertAndGetWiderMatch (
                     if (candidateDist > distanceToNextMatch) {
                         distanceToNextMatch = candidateDist;
                         matchChainPos = pos;
-                    }
-                }
+                }   }
                 if (distanceToNextMatch > 1) {
                     if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
                     matchIndex -= distanceToNextMatch;
                     continue;
-                }
-            }
-        }
+        }   }   }
 
         {   U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex);
             if (patternAnalysis && distNextMatch==1 && matchChainPos==0) {
@@ -309,7 +306,7 @@ LZ4HC_InsertAndGetWiderMatch (
                     if ( ((pattern & 0xFFFF) == (pattern >> 16))
                       &  ((pattern & 0xFF)   == (pattern >> 24)) ) {
                         repeat = rep_confirmed;
-                        srcPatternLength = LZ4HC_countPattern(ip+4, iHighLimit, pattern) + 4;
+                        srcPatternLength = LZ4HC_countPattern(ip+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
                     } else {
                         repeat = rep_not;
                 }   }
@@ -318,19 +315,29 @@ LZ4HC_InsertAndGetWiderMatch (
                     const BYTE* const matchPtr = base + matchCandidateIdx;
                     if (LZ4_read32(matchPtr) == pattern) {  /* good candidate */
                         size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
-                        const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
-                        size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern);
+                        const BYTE* const lowestMatchPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
+                        size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern);
                         size_t const currentSegmentLength = backLength + forwardPatternLength;
 
                         if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
                           && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
                             matchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
                         } else {
-                            matchIndex = matchCandidateIdx - (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
-                        }
+                            matchIndex = matchCandidateIdx - (U32)backLength;   /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */
+                            if (lookBackLength==0) {  /* no back possible */
+                                size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
+                                if ((size_t)longest < maxML) {
+                                    longest = maxML;
+                                    *matchpos = base + matchIndex;   /* virtual pos, relative to ip, to retrieve offset */
+                                    *startpos = ip;
+                                }
+                                {   U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex);
+                                    if (distToNextPattern > matchIndex) break;  /* avoid overflow */
+                                    matchIndex -= distToNextPattern;
+                        }   }   }
                         continue;
                 }   }
-        }   }
+        }   }   /* PA optimization */
 
         /* follow current chain */
         matchIndex -= DELTANEXTU16(chainTable, matchIndex+matchChainPos);
-- 
cgit v0.12


From 200b2960d5a5d0fe76d34bb826c236c7c2941563 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 6 May 2018 18:26:14 -0700
Subject: fixed minor conversion warning

---
 lib/lz4hc.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 059c4b5..b89983f 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -327,7 +327,8 @@ LZ4HC_InsertAndGetWiderMatch (
                             if (lookBackLength==0) {  /* no back possible */
                                 size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
                                 if ((size_t)longest < maxML) {
-                                    longest = maxML;
+                                    assert(maxML < 2 GB);
+                                    longest = (int)maxML;
                                     *matchpos = base + matchIndex;   /* virtual pos, relative to ip, to retrieve offset */
                                     *startpos = ip;
                                 }
-- 
cgit v0.12


From ba1c7148a56029aa01795bfcf8cc365a2551f968 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 7 May 2018 12:14:26 -0700
Subject: renamed variable for clarity

---
 lib/lz4hc.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b89983f..8108ea0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -243,7 +243,7 @@ LZ4HC_InsertAndGetWiderMatch (
                 matchIndex, lowestMatchIndex);
 
     while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
-        int mlt=0;
+        int matchLength=0;
         nbAttempts--;
         assert(matchIndex < ipIndex);
         if (favorDecSpeed && (ipIndex - matchIndex < 8)) {
@@ -256,10 +256,10 @@ LZ4HC_InsertAndGetWiderMatch (
             if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
                     int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
-                    mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-                    mlt -= back;
-                    if (mlt > longest) {
-                        longest = mlt;
+                    matchLength = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+                    matchLength -= back;
+                    if (matchLength > longest) {
+                        longest = matchLength;
                         *matchpos = matchPtr + back;
                         *startpos = ip + back;
             }   }   }
@@ -270,18 +270,18 @@ LZ4HC_InsertAndGetWiderMatch (
                 int back = 0;
                 const BYTE* vLimit = ip + (dictLimit - matchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
-                mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
-                if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
-                    mlt += LZ4_count(ip+mlt, lowPrefixPtr, iHighLimit);
+                matchLength = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
+                if ((ip+matchLength == vLimit) && (vLimit < iHighLimit))
+                    matchLength += LZ4_count(ip+matchLength, lowPrefixPtr, iHighLimit);
                 back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0;
-                mlt -= back;
-                if (mlt > longest) {
-                    longest = mlt;
+                matchLength -= back;
+                if (matchLength > longest) {
+                    longest = matchLength;
                     *matchpos = base + matchIndex + back;   /* virtual pos, relative to ip, to retrieve offset */
                     *startpos = ip + back;
         }   }   }
 
-        if (chainSwap && mlt==longest) {    /* better match => select a better chain */
+        if (chainSwap && matchLength==longest) {    /* better match => select a better chain */
             assert(lookBackLength==0);   /* search forward only */
             if (matchIndex + longest <= ipIndex) {
                 U32 distanceToNextMatch = 1;
-- 
cgit v0.12