From 8c006b19bbed0b3bb2466dc8af2a603ca0ba7fab Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 29 Jan 2018 13:20:16 -0500
Subject: Add a Benchmarking Tool For Compression with Context Re-Use

---
 tests/Makefile     |   8 +-
 tests/framebench.c | 293 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 299 insertions(+), 2 deletions(-)
 create mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index 5954370..77d9e36 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -62,7 +62,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen
+all: fullbench fuzzer frametest datagen framebench
 
 all32: CFLAGS+=-m32
 all32: all
@@ -98,6 +98,9 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
+framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -109,7 +112,8 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
-        fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
+        framebench$(EXT) \
+        fasttest$(EXT) datagen$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
 
diff --git a/tests/framebench.c b/tests/framebench.c
new file mode 100644
index 0000000..8dcfa41
--- /dev/null
+++ b/tests/framebench.c
@@ -0,0 +1,293 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "lz4.h"
+#include "lz4frame.h"
+#include "lz4frame_static.h"
+
+#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
+
+typedef struct {
+  size_t iter;
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  char *obuf;
+  size_t osize;
+  const char* ibuf;
+  size_t isize;
+  size_t num_ibuf;
+  const LZ4F_CDict* cdict;
+  LZ4F_preferences_t* prefs;
+  const LZ4F_compressOptions_t* options;
+} bench_params_t;
+
+size_t compress_frame(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressFrame_usingCDict(
+    obuf,
+    osize,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    cdict,
+    prefs);
+  LZ4F_CHECK(oused);
+
+  return oused;
+}
+
+size_t compress_begin(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+  const LZ4F_compressOptions_t* options = p->options;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressUpdate(
+    cctx,
+    obuf,
+    oend - obuf,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    options);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
+  LZ4F_CHECK(oused);
+
+  return obuf - p->obuf;
+}
+
+size_t compress_default(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_extState(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4_stream_t *ctx = p->ctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_fast_extState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+uint64_t bench(
+    size_t (*fun)(bench_params_t *),
+    uint64_t repetitions,
+    bench_params_t *params,
+    size_t *osizePtr
+) {
+  struct timespec start, end;
+  size_t i, osize = 0;
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
+
+  for (i = 0; i < repetitions; i++) {
+    size_t o;
+    params->iter = i;
+    o = fun(params);
+    if (!o) return 0;
+    osize += o;
+  }
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+
+  *osizePtr = osize / repetitions;
+  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+}
+
+int main(int argc, char *argv[]) {
+
+
+  struct stat st;
+  size_t bytes_read;
+
+  const char *dict_fn;
+  size_t dict_size;
+  char *dict_buf;
+  FILE *dict_file;
+
+  const char *in_fn;
+  size_t in_size;
+  size_t num_in_buf;
+  size_t cur_in_buf;
+  char *in_buf;
+  FILE *in_file;
+
+  size_t out_size;
+  char *out_buf;
+
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  LZ4F_CDict *cdict;
+  LZ4F_preferences_t prefs;
+  LZ4F_compressOptions_t options;
+
+  size_t out_used;
+
+  uint64_t time_taken;
+  uint64_t repetitions;
+
+  bench_params_t params;
+
+  if (argc != 3) return 1;
+  dict_fn = argv[1];
+  in_fn = argv[2];
+
+  if (stat(dict_fn, &st)) return 1;
+  dict_size = st.st_size;
+  dict_buf = (char *)malloc(dict_size);
+  if (!dict_buf) return 1;
+  dict_file = fopen(dict_fn, "r");
+  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
+  if (bytes_read != dict_size) return 1;
+
+  if (stat(in_fn, &st)) return 1;
+  in_size = st.st_size;
+  num_in_buf = 256 * 1024 * 1024 / in_size;
+  if (num_in_buf == 0) {
+    num_in_buf = 1;
+  }
+
+  in_buf = (char *)malloc(in_size * num_in_buf);
+  if (!in_buf) return 1;
+  in_file = fopen(in_fn, "r");
+  bytes_read = fread(in_buf, 1, in_size, in_file);
+  if (bytes_read != in_size) return 1;
+
+  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
+    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
+  }
+
+  if (in_size <= 1024) {
+    repetitions = 100000;
+  } else
+  if (in_size <= 16384) {
+    repetitions = 10000;
+  } else
+  if (in_size <= 131072) {
+    repetitions = 1000;
+  } else
+  if (in_size <= 1048576) {
+    repetitions = 100;
+  } else {
+    repetitions = 50;
+  }
+
+  memset(&prefs, 0, sizeof(prefs));
+  prefs.autoFlush = 1;
+  if (in_size < 64 * 1024)
+    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
+  prefs.frameInfo.contentSize = in_size;
+
+  memset(&options, 0, sizeof(options));
+  options.stableSrc = 1;
+
+  out_size = LZ4F_compressFrameBound(in_size, &prefs);
+  out_buf = (char *)malloc(out_size);
+  if (!out_buf) return 1;
+
+  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
+  if (cctx == NULL) return 1;
+
+  ctx = LZ4_createStream();
+  if (ctx == NULL) return 1;
+
+  cdict = LZ4F_createCDict(dict_buf, dict_size);
+  if (!cdict) return 1;
+
+  fprintf(stderr, "dict  size: %zd\n", dict_size);
+  fprintf(stderr, "input size: %zd\n", in_size);
+
+  params.ctx = ctx;
+  params.cctx = cctx;
+  params.obuf = out_buf;
+  params.osize = out_size;
+  params.ibuf = in_buf;
+  params.isize = in_size;
+  params.num_ibuf = num_in_buf;
+  params.cdict = NULL;
+  params.prefs = &prefs;
+  params.options = &options;
+
+  time_taken = bench(compress_default, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_extState, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  params.cdict = cdict;
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  return 0;
+}
\ No newline at end of file
-- 
cgit v0.12


From 6d156fea56debbde1699b54a52c4907ff0b96017 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Oct 2017 16:13:33 -0400
Subject: Allow Empty Dictionaries

---
 lib/lz4.c | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 38a865f..898b82a 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1018,12 +1018,6 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     if ((dict->initCheck) || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
         LZ4_resetStream(LZ4_dict);
 
-    if (dictSize < (int)HASH_UNIT) {
-        dict->dictionary = NULL;
-        dict->dictSize = 0;
-        return 0;
-    }
-
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
     dict->currentOffset += 64 KB;
     base = p - dict->currentOffset;
@@ -1031,6 +1025,10 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     dict->dictSize = (U32)(dictEnd - p);
     dict->currentOffset += dict->dictSize;
 
+    if (dictSize < (int)HASH_UNIT) {
+        return 0;
+    }
+
     while (p <= dictEnd-HASH_UNIT) {
         LZ4_putPosition(p, dict->hashTable, byU32, base);
         p+=3;
-- 
cgit v0.12


From 6933f5ad9ca57caa8a76103f1382560fba0f9691 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 17 Oct 2017 15:23:02 -0400
Subject: Remove Obsolete Stream Functions to Free Space in LZ4_stream_t

---
 lib/lz4.c | 31 -------------------------------
 lib/lz4.h |  6 ------
 2 files changed, 37 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 898b82a..6168a2a 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1515,37 +1515,6 @@ int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4
 int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
 
 
-/* Obsolete Streaming functions */
-
-int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
-
-static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base)
-{
-    MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t));
-    lz4ds->internal_donotuse.bufferStart = base;
-}
-
-int LZ4_resetStreamState(void* state, char* inputBuffer)
-{
-    if ((((uptrval)state) & 3) != 0) return 1;   /* Error : pointer is not aligned on 4-bytes boundary */
-    LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer);
-    return 0;
-}
-
-void* LZ4_create (char* inputBuffer)
-{
-    LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t));
-    LZ4_init (lz4ds, (BYTE*)inputBuffer);
-    return lz4ds;
-}
-
-char* LZ4_slideInputBuffer (void* LZ4_Data)
-{
-    LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse;
-    int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB);
-    return (char*)(ctx->bufferStart + dictSize);
-}
-
 /* Obsolete streaming decompression functions */
 
 int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
diff --git a/lib/lz4.h b/lib/lz4.h
index 911ea79..e5d4039 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -465,12 +465,6 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
 LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
-/* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
-LZ4_DEPRECATED("use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
-
 /* Obsolete streaming decoding functions */
 LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
 LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
-- 
cgit v0.12


From 5709891de6513ee8253f27e8d3207ec1fdcff14b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:11:35 -0500
Subject: Add a Table Type Field to LZ4_stream_t

---
 lib/lz4.c | 24 ++++++++++++++++++------
 lib/lz4.h |  6 ++++--
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 6168a2a..7b8de77 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -446,7 +446,7 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 *  Local Structures and types
 **************************************/
 typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
-typedef enum { byPtr, byU32, byU16 } tableType_t;
+typedef enum { unusedTable = 0, byPtr = 1, byU32 = 2, byU16 = 3 } tableType_t;
 
 typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive;
 typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
@@ -496,6 +496,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy
 {
     switch (tableType)
     {
+    case unusedTable: { /* illegal! */ assert(0); return; }
     case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; }
     case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; }
     case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; }
@@ -551,6 +552,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
 
+    ptrdiff_t retval = 0;
+
     U32 forwardH;
 
     /* Init conditions */
@@ -622,7 +625,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             token = op++;
             if ((outputLimited) &&  /* Check output buffer overflow */
                 (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
-                return 0;
+                goto _clean_up;
             if (litLength >= RUN_MASK) {
                 int len = (int)litLength-RUN_MASK;
                 *token = (RUN_MASK<<ML_BITS);
@@ -662,7 +665,7 @@ _next_match:
 
             if ( outputLimited &&    /* Check output buffer overflow */
                 (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
-                return 0;
+                goto _clean_up;
             if (matchCode >= ML_MASK) {
                 *token += ML_MASK;
                 matchCode -= ML_MASK;
@@ -711,7 +714,7 @@ _last_literals:
     {   size_t const lastRun = (size_t)(iend - anchor);
         if ( (outputLimited) &&  /* Check output buffer overflow */
             ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) )
-            return 0;
+            goto _clean_up;
         if (lastRun >= RUN_MASK) {
             size_t accumulator = lastRun - RUN_MASK;
             *op++ = RUN_MASK << ML_BITS;
@@ -724,8 +727,13 @@ _last_literals:
         op += lastRun;
     }
 
+    retval = (((char*)op)-dest);
+
+_clean_up:
+    cctx->tableType = tableType;
+
     /* End */
-    return (int) (((char*)op)-dest);
+    return (int)retval;
 }
 
 
@@ -997,6 +1005,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(4, "LZ4_resetStream");
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
+    LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
@@ -1015,7 +1024,9 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
-    if ((dict->initCheck) || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
+    if ((dict->initCheck)
+      || (dict->tableType != byU32 && dict->tableType != unusedTable)
+      || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
         LZ4_resetStream(LZ4_dict);
 
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
@@ -1024,6 +1035,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     dict->dictionary = p;
     dict->dictSize = (U32)(dictEnd - p);
     dict->currentOffset += dict->dictSize;
+    dict->tableType = byU32;
 
     if (dictSize < (int)HASH_UNIT) {
         return 0;
diff --git a/lib/lz4.h b/lib/lz4.h
index e5d4039..d2e2103 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -360,7 +360,8 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 typedef struct {
     uint32_t hashTable[LZ4_HASH_SIZE_U32];
     uint32_t currentOffset;
-    uint32_t initCheck;
+    uint16_t initCheck;
+    uint16_t tableType;
     const uint8_t* dictionary;
     uint8_t* bufferStart;   /* obsolete, used for slideInputBuffer */
     uint32_t dictSize;
@@ -378,7 +379,8 @@ typedef struct {
 typedef struct {
     unsigned int hashTable[LZ4_HASH_SIZE_U32];
     unsigned int currentOffset;
-    unsigned int initCheck;
+    unsigned short initCheck;
+    unsigned short tableType;
     const unsigned char* dictionary;
     unsigned char* bufferStart;   /* obsolete, used for slideInputBuffer */
     unsigned int dictSize;
-- 
cgit v0.12


From f34fb3c42dbb824af97800eefdb487dff31e0c13 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:25:29 -0500
Subject: Add Bounds Check to locateBuffDiff

---
 tests/frametest.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/frametest.c b/tests/frametest.c
index 88d0afd..986a16d 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -730,15 +730,17 @@ _output_error:
 
 static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
 {
-    int p=0;
+    size_t p=0;
     const BYTE* b1=(const BYTE*)buff1;
     const BYTE* b2=(const BYTE*)buff2;
     if (nonContiguous) {
         DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size);
         return;
     }
-    while (b1[p]==b2[p]) p++;
-    DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]);
+    while (p < size && b1[p]==b2[p]) p++;
+    if (p != size) {
+        DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]);
+    }
 }
 
 
-- 
cgit v0.12


From d6a3024dbb6375190e6e02ad8b06aae28e76b946 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:41:11 -0500
Subject: Add LZ4_compress_fast_safeExtState Function

---
 lib/lz4.c | 10 +++++++++-
 lib/lz4.h |  8 ++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 7b8de77..45cc7c9 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -739,9 +739,17 @@ _clean_up:
 
 int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
+  LZ4_resetStream((LZ4_stream_t*)state);
+  return LZ4_compress_fast_safeExtState(state, source, dest, inputSize, maxOutputSize, acceleration);
+}
+
+
+int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+{
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
-    LZ4_resetStream((LZ4_stream_t*)state);
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
+    ctx->dictionary = NULL;
+    ctx->dictSize = 0;
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit)
diff --git a/lib/lz4.h b/lib/lz4.h
index d2e2103..0013ffe 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -179,13 +179,21 @@ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int d
 
 
 /*!
+LZ4_compress_fast_safeExtState() :
 LZ4_compress_fast_extState() :
     Same compression function, just using an externally allocated memory space to store compression state.
     Use LZ4_sizeofState() to know how much memory must be allocated,
     and allocate it on 8-bytes boundaries (using malloc() typically).
     Then, provide it as 'void* state' to compression function.
+
+    Use _safeExtState variant if LZ4_resetStream() was called on the state
+    buffer before being used for the first time (calls to this function leave
+    the state in a safe state, so zeroing is not required between calls).
+    Otherwise, using legacy _extState requires LZ4 to reinitialize the state
+    internally for every call.
 */
 LZ4LIB_API int LZ4_sizeofState(void);
+LZ4LIB_API int LZ4_compress_fast_safeExtState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
-- 
cgit v0.12


From aa36e118f10d825ceb3899d3e137d643ed469331 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:45:59 -0500
Subject: Const-ify Table Arg to LZ4_getPosition(OnHash)

---
 lib/lz4.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 45cc7c9..46935f8 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -509,14 +509,14 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_
     LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
 }
 
-static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
-    if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
-    if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
-    { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
+    if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; }
+    if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; return hashTable[h] + srcBase; }
+    { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
 }
 
-LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     U32 const h = LZ4_hashPosition(p, tableType);
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
-- 
cgit v0.12


From b3628cb0c51bb93f681707b2d0cdbce5ccdae818 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:09:38 -0500
Subject: Avoid Resetting the Context When Possible

---
 lib/lz4.c          | 61 ++++++++++++++++++++++++++++++++++++++----------------
 lib/lz4frame.c     | 21 +++++++++++++++++--
 tests/framebench.c |  2 +-
 3 files changed, 63 insertions(+), 21 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 46935f8..d558b3b 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -538,7 +538,20 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                  const U32 acceleration)
 {
     const BYTE* ip = (const BYTE*) source;
-    const BYTE* base;
+
+    /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
+     * therefore safe to use no matter what mode we're in. Otherwise, we figure
+     * out if it's safe to leave as is or whether it needs to be reset.
+     */
+    const int resetTable = cctx->tableType != unusedTable && (
+                           cctx->tableType != tableType ||
+                            (tableType == byU16 &&
+                             cctx->currentOffset + inputSize >= 0xFFFFU) ||
+                           tableType == byPtr);
+
+    size_t currentOffset = ((tableType == byU32 || tableType == byU16) &&
+                            !resetTable) ? cctx->currentOffset : 0;
+    const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
     const BYTE* const lowRefLimit = ip - cctx->dictSize;
     const BYTE* const dictionary = cctx->dictionary;
@@ -556,17 +569,18 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     U32 forwardH;
 
+    /* TODO: resurrect dictIssue check */
+    (void)dictIssue;
+
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     switch(dict)
     {
     case noDict:
     default:
-        base = (const BYTE*)source;
         lowLimit = (const BYTE*)source;
         break;
     case withPrefix64k:
-        base = (const BYTE*)source - cctx->currentOffset;
         lowLimit = (const BYTE*)source - cctx->dictSize;
         break;
     case usingExtDict:
@@ -575,6 +589,13 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         break;
     }
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
+
+    if (resetTable) {
+      DEBUGLOG(4, "Resetting table in %p", cctx);
+      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
+      cctx->currentOffset = 0;
+    }
+
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* First Byte */
@@ -612,7 +633,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
 
-            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
+            } while ( match < lowRefLimit
                 || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
         }
@@ -700,7 +721,7 @@ _next_match:
                 lowLimit = (const BYTE*)source;
         }   }
         LZ4_putPosition(ip, cctx->hashTable, tableType, base);
-        if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
+        if ( match >= lowRefLimit
             && (match+MAX_DISTANCE>=ip)
             && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
         { token=op++; *token=0; goto _next_match; }
@@ -730,6 +751,8 @@ _last_literals:
     retval = (((char*)op)-dest);
 
 _clean_up:
+    cctx->currentOffset += (U32)inputSize;
+    cctx->dictSize += (U32)inputSize;
     cctx->tableType = tableType;
 
     /* End */
@@ -767,14 +790,16 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
 
 int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
+    int result;
 #if (LZ4_HEAPMODE)
-    void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    ctxPtr->internal_donotuse.currentOffset = 0;
 #else
     LZ4_stream_t ctx;
-    void* const ctxPtr = &ctx;
+    LZ4_stream_t* const ctxPtr = &ctx;
 #endif
-
-    int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
+    LZ4_resetStream(ctxPtr);
+    result = LZ4_compress_fast_safeExtState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
 
 #if (LZ4_HEAPMODE)
     FREEMEM(ctxPtr);
@@ -1005,20 +1030,23 @@ LZ4_stream_t* LZ4_createStream(void)
 {
     LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64);
     LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal));    /* A compilation error here means LZ4_STREAMSIZE is not large enough */
+    DEBUGLOG(4, "LZ4_createStream %p", lz4s);
     LZ4_resetStream(lz4s);
     return lz4s;
 }
 
 void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
-    DEBUGLOG(4, "LZ4_resetStream");
+    DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
+    LZ4_stream->internal_donotuse.currentOffset = 0;
     LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
 {
     if (!LZ4_stream) return 0;   /* support free on NULL */
+    DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream);
     FREEMEM(LZ4_stream);
     return (0);
 }
@@ -1032,6 +1060,8 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
+    DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
+
     if ((dict->initCheck)
       || (dict->tableType != byU32 && dict->tableType != unusedTable)
       || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
@@ -1066,6 +1096,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
         U32 const delta = LZ4_dict->currentOffset - 64 KB;
         const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
         int i;
+        DEBUGLOG(4, "LZ4_renormDictT %p", LZ4_dict);
         for (i=0; i<LZ4_HASH_SIZE_U32; i++) {
             if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0;
             else LZ4_dict->hashTable[i] -= delta;
@@ -1100,14 +1131,10 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
-        int result;
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
         else
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
-        streamPtr->dictSize += (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
-        return result;
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
     }
 
     /* external dictionary mode */
@@ -1118,7 +1145,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
             result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
         streamPtr->dictionary = (const BYTE*)source;
         streamPtr->dictSize = (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
         return result;
     }
 }
@@ -1139,7 +1165,6 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
 
     streamPtr->dictionary = (const BYTE*)source;
     streamPtr->dictSize = (U32)inputSize;
-    streamPtr->currentOffset += (U32)inputSize;
 
     return result;
 }
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a394d1f..bf70b5c 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -354,6 +354,7 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
         prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* only one block => no need for inter-block link */
 
     if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+        LZ4_resetStream(&lz4ctx);
         cctxI.lz4CtxPtr = &lz4ctx;
         cctxI.lz4CtxLevel = 1;
     }  /* fast compression context pre-created on stack */
@@ -521,7 +522,14 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
             if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             cctxPtr->lz4CtxLevel = ctxTypeID;
-    }   }
+        } else if (cctxPtr->lz4CtxLevel != ctxTypeID) {
+            /* otherwise, we must be transitioning from HC -> LZ4.
+             * In that case, avoid reallocating, since a LZ4 ctx
+             * fits in an HC ctx. Just reset. */
+            LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
+            cctxPtr->lz4CtxLevel = ctxTypeID;
+        }
+    }
 
     /* Buffer Management */
     if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
@@ -654,11 +662,20 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
+    LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)ctx)->internal_donotuse;
+    assert(!internal_ctx->initCheck);
+    if (internal_ctx->currentOffset > 1 GB) {
+        /* Init the context */
+        LZ4_resetStream((LZ4_stream_t*)ctx);
+    }
+    /* Clear any local dictionary */
+    internal_ctx->dictionary = NULL;
+    internal_ctx->dictSize = 0;
     if (cdict) {
         memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
-    return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration);
+    return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
 static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
diff --git a/tests/framebench.c b/tests/framebench.c
index 8dcfa41..21c3704 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -118,7 +118,7 @@ size_t compress_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_extState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  oused = LZ4_compress_fast_safeExtState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
   obuf += oused;
 
   return obuf - p->obuf;
-- 
cgit v0.12


From 7060bcabf041395d6249a701729d02e28529bf82 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 14:44:53 -0500
Subject: Only Re-Alloc / Reset When Needed When Switching Between Regular and
 High Compression Modes

---
 lib/lz4frame.c | 40 +++++++++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index bf70b5c..cd61925 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -188,7 +188,15 @@ typedef struct LZ4F_cctx_s
     U64    totalInSize;
     XXH32_state_t xxh;
     void*  lz4CtxPtr;
-    U32    lz4CtxLevel;   /* 0: unallocated;  1: LZ4_stream_t;  3: LZ4_streamHC_t */
+    /* lz4CtxLevel records both what size of memory has been allocated into the
+     * ctx pointer field and what it's being used as.
+     * - The low bit (& 1) value indicates whether the ctx is being used for
+     *   hc (1) or not (0).
+     * - The next bit (& 2) indicates whether the allocated memory is big
+     *   enough for a non-hc context.
+     * - The next bit (& 4) indicates whether the allocated memory is big
+     *   enough for an hc context. */
+    U32    lz4CtxLevel;
 } LZ4F_cctx_t;
 
 
@@ -356,7 +364,7 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
     if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
         LZ4_resetStream(&lz4ctx);
         cctxI.lz4CtxPtr = &lz4ctx;
-        cctxI.lz4CtxLevel = 1;
+        cctxI.lz4CtxLevel = 2;
     }  /* fast compression context pre-created on stack */
 
     memset(&options, 0, sizeof(options));
@@ -513,21 +521,27 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->prefs = *preferencesPtr;
 
     /* Ctx Management */
-    {   U32 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;  /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
-        if (cctxPtr->lz4CtxLevel < ctxTypeID) {
+    {   U32 const ctxSufficientAllocBits = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 2 : 6;
+        U32 const ctxTypeBit = cctxPtr->prefs.compressionLevel >= LZ4HC_CLEVEL_MIN;
+        if ((cctxPtr->lz4CtxLevel & ctxSufficientAllocBits) != ctxSufficientAllocBits) {
             FREEMEM(cctxPtr->lz4CtxPtr);
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
-            else
+            } else {
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
+            }
             if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
-            cctxPtr->lz4CtxLevel = ctxTypeID;
-        } else if (cctxPtr->lz4CtxLevel != ctxTypeID) {
-            /* otherwise, we must be transitioning from HC -> LZ4.
-             * In that case, avoid reallocating, since a LZ4 ctx
-             * fits in an HC ctx. Just reset. */
-            LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
-            cctxPtr->lz4CtxLevel = ctxTypeID;
+            cctxPtr->lz4CtxLevel = ctxSufficientAllocBits | ctxTypeBit;
+        } else if ((cctxPtr->lz4CtxLevel & 1) != ctxTypeBit) {
+            /* otherwise, a sufficient buffer is allocated, but we need to
+             * reset it to the correct context type */
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+                LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
+                cctxPtr->lz4CtxLevel &= ~1;
+            } else {
+                LZ4_resetStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+                cctxPtr->lz4CtxLevel |= 1;
+            }
         }
     }
 
-- 
cgit v0.12


From 62cb52b3410bdfd696669910de010ab03666edc8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:10:15 -0500
Subject: Initialize Current Offset to 1

---
 lib/lz4.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d558b3b..dde27e9 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -550,7 +550,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                            tableType == byPtr);
 
     size_t currentOffset = ((tableType == byU32 || tableType == byU16) &&
-                            !resetTable) ? cctx->currentOffset : 0;
+                            !resetTable) ? cctx->currentOffset : 1;
     const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
     const BYTE* const lowRefLimit = ip - cctx->dictSize;
@@ -593,7 +593,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     if (resetTable) {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 0;
+      cctx->currentOffset = 1;
     }
 
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
@@ -793,7 +793,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     int result;
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
-    ctxPtr->internal_donotuse.currentOffset = 0;
+    ctxPtr->internal_donotuse.currentOffset = 1;
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1039,7 +1039,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-    LZ4_stream->internal_donotuse.currentOffset = 0;
+    LZ4_stream->internal_donotuse.currentOffset = 1;
     LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 
-- 
cgit v0.12


From 73cc39327e3abc28a360323c4f26c3c34d87ff07 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:18:24 -0500
Subject: Lookup Matches in Separate Dictionary Context

---
 lib/lz4.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++------------
 lib/lz4.h | 14 +++++++------
 2 files changed, 65 insertions(+), 19 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index dde27e9..40cac94 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -448,7 +448,7 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
 typedef enum { unusedTable = 0, byPtr = 1, byU32 = 2, byU16 = 3 } tableType_t;
 
-typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive;
+typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingExtDictCtx } dict_directive;
 typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
 
 typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
@@ -533,7 +533,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                  const int maxOutputSize,
                  const limitedOutput_directive outputLimited,
                  const tableType_t tableType,
-                 const dict_directive dict,
+                 const dict_directive dictDirective,
                  const dictIssue_directive dictIssue,
                  const U32 acceleration)
 {
@@ -553,15 +553,27 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                             !resetTable) ? cctx->currentOffset : 1;
     const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
-    const BYTE* const lowRefLimit = ip - cctx->dictSize;
-    const BYTE* const dictionary = cctx->dictionary;
-    const BYTE* const dictEnd = dictionary + cctx->dictSize;
-    const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source;
+
+    const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
+    const BYTE* const dictionary =
+        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictionary;
+    const U32 dictSize =
+        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictSize;
+
+    const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
+    const BYTE* const dictEnd = dictionary + dictSize;
     const BYTE* anchor = (const BYTE*) source;
     const BYTE* const iend = ip + inputSize;
     const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1;
     const BYTE* const matchlimit = iend - LASTLITERALS;
 
+    /* the dictCtx currentOffset is indexed on the start of the dictionary,
+     * while a dictionary in the current context precedes the currentOffset */
+    const BYTE* dictBase = dictDirective == usingExtDictCtx ?
+        (const BYTE*) source - dictCtx->currentOffset :
+        (const BYTE*) source - dictSize - currentOffset;
+    const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
+
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
 
@@ -574,17 +586,19 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
-    switch(dict)
+    switch(dictDirective)
     {
     case noDict:
     default:
         lowLimit = (const BYTE*)source;
         break;
     case withPrefix64k:
-        lowLimit = (const BYTE*)source - cctx->dictSize;
+        lowLimit = (const BYTE*)source - dictSize;
         break;
     case usingExtDict:
-        base = (const BYTE*)source - cctx->currentOffset;
+        lowLimit = (const BYTE*)source;
+        break;
+    case usingExtDictCtx:
         lowLimit = (const BYTE*)source;
         break;
     }
@@ -622,8 +636,19 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 assert(ip < mflimitPlusOne);
 
                 match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
-                if (dict==usingExtDict) {
+                if (dictDirective == usingExtDictCtx) {
                     if (match < (const BYTE*)source) {
+                        /* there was no match, try the dictionary */
+                        /* TODO: use precalc-ed hash? */
+                        match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
+                        refDelta = dictDelta;
+                        lowLimit = dictionary;
+                    } else {
+                        refDelta = 0;
+                        lowLimit = (const BYTE*)source;
+                    }
+                } else if (dictDirective==usingExtDict) {
+                    if (match < (const BYTE*)source && dictionary != NULL) {
                         refDelta = dictDelta;
                         lowLimit = dictionary;
                     } else {
@@ -667,7 +692,7 @@ _next_match:
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dict==usingExtDict) && (lowLimit==dictionary)) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictionary)) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
@@ -712,8 +737,19 @@ _next_match:
 
         /* Test next position */
         match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
-        if (dict==usingExtDict) {
+        if (dictDirective == usingExtDictCtx) {
             if (match < (const BYTE*)source) {
+                /* there was no match, try the dictionary */
+                /* TODO: use precalc-ed hash? */
+                match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
+                refDelta = dictDelta;
+                lowLimit = dictionary;
+            } else {
+                refDelta = 0;
+                lowLimit = (const BYTE*)source;
+            }
+        } else if (dictDirective==usingExtDict) {
+            if (match < (const BYTE*)source && dictionary != NULL) {
                 refDelta = dictDelta;
                 lowLimit = dictionary;
             } else {
@@ -751,8 +787,15 @@ _last_literals:
     retval = (((char*)op)-dest);
 
 _clean_up:
+    if (dictDirective == usingExtDictCtx) {
+      /* Subsequent linked blocks can't use the dictionary. */
+      /* Instead, they use the block we just compressed. */
+      cctx->dictCtx = NULL;
+      cctx->dictSize = (U32)inputSize;
+    } else {
+      cctx->dictSize += (U32)inputSize;
+    }
     cctx->currentOffset += (U32)inputSize;
-    cctx->dictSize += (U32)inputSize;
     cctx->tableType = tableType;
 
     /* End */
@@ -773,6 +816,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
     ctx->dictionary = NULL;
     ctx->dictSize = 0;
+    ctx->dictCtx = NULL;
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit)
diff --git a/lib/lz4.h b/lib/lz4.h
index 0013ffe..58a1e39 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -365,15 +365,16 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #include <stdint.h>
 
-typedef struct {
+typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
+struct LZ4_stream_t_internal {
     uint32_t hashTable[LZ4_HASH_SIZE_U32];
     uint32_t currentOffset;
     uint16_t initCheck;
     uint16_t tableType;
     const uint8_t* dictionary;
-    uint8_t* bufferStart;   /* obsolete, used for slideInputBuffer */
+    const LZ4_stream_t_internal* dictCtx;
     uint32_t dictSize;
-} LZ4_stream_t_internal;
+};
 
 typedef struct {
     const uint8_t* externalDict;
@@ -384,15 +385,16 @@ typedef struct {
 
 #else
 
-typedef struct {
+typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
+struct LZ4_stream_t_internal {
     unsigned int hashTable[LZ4_HASH_SIZE_U32];
     unsigned int currentOffset;
     unsigned short initCheck;
     unsigned short tableType;
     const unsigned char* dictionary;
-    unsigned char* bufferStart;   /* obsolete, used for slideInputBuffer */
+    const LZ4_stream_t_internal* dictCtx;
     unsigned int dictSize;
-} LZ4_stream_t_internal;
+};
 
 typedef struct {
     const unsigned char* externalDict;
-- 
cgit v0.12


From 68c6bd17b82a0be501237af682f6472b0d39114f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:19:13 -0500
Subject: Set Dictionary Context Pointer Rather than Copying the Context In

---
 lib/lz4.c      | 17 +++++++++++++----
 lib/lz4frame.c | 35 +++++++++++++++++++++++++++++------
 2 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 40cac94..7d1d65b 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1183,10 +1183,19 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* external dictionary mode */
     {   int result;
-        if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
-        else
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
+        if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
+            if (streamPtr->dictCtx) {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, dictSmall, acceleration);
+            } else {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
+            }
+        } else {
+            if (streamPtr->dictCtx) {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, noDictIssue, acceleration);
+            } else {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
+            }
+        }
         streamPtr->dictionary = (const BYTE*)source;
         streamPtr->dictSize = (U32)inputSize;
         return result;
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index cd61925..9ff7766 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -571,16 +571,36 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         /* frame init only for blockLinked : blockIndependent will be init at each block */
         if (cdict) {
             if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-                memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
+                assert(!internal_ctx->initCheck);
+                if (internal_ctx->currentOffset > 1 GB) {
+                    /* Init the context */
+                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
+                }
+                /* Clear any local dictionary */
+                internal_ctx->dictionary = NULL;
+                internal_ctx->dictSize = 0;
+                /* Point to the dictionary context */
+                internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
             } else {
                 memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
                 LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
             }
         } else {
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-                LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
-            else
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
+                assert(!internal_ctx->initCheck);
+                if (internal_ctx->currentOffset > 1 GB) {
+                    /* Init the context */
+                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
+                }
+                /* Clear any local dictionary */
+                internal_ctx->dictionary = NULL;
+                internal_ctx->dictSize = 0;
+                internal_ctx->dictCtx = NULL;
+            } else {
                 LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
+            }
         }
     }
 
@@ -686,10 +706,13 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize
     internal_ctx->dictionary = NULL;
     internal_ctx->dictSize = 0;
     if (cdict) {
-        memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
+        /* Point to the dictionary context */
+        internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
+    } else {
+        internal_ctx->dictCtx = NULL;
+        return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
-    return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
 static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
-- 
cgit v0.12


From cea09d67a9deca8370dc634fd6c57437114cacbf Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 29 Jan 2018 17:09:52 -0500
Subject: Hoist Table Reset One Level Up

---
 lib/lz4.c | 86 +++++++++++++++++++++++++++++++++++++++------------------------
 1 file changed, 53 insertions(+), 33 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 7d1d65b..1f0450e 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -523,6 +523,29 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
 }
 
 
+LZ4_FORCE_INLINE void LZ4_resetTable(
+        LZ4_stream_t_internal* const cctx,
+        const int inputSize,
+        const tableType_t tableType) {
+  /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
+   * therefore safe to use no matter what mode we're in. Otherwise, we figure
+   * out if it's safe to leave as is or whether it needs to be reset.
+   */
+  if (cctx->tableType != unusedTable && (
+      cctx->tableType != tableType ||
+      (tableType == byU16 &&
+       cctx->currentOffset + inputSize >= 0xFFFFU) ||
+      (tableType == byU32 &&
+       cctx->currentOffset > 1 GB) ||
+      tableType == byPtr))
+  {
+      DEBUGLOG(4, "Resetting table in %p", cctx);
+      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
+      cctx->currentOffset = 1;
+      cctx->tableType = unusedTable;
+  }
+}
+
 /** LZ4_compress_generic() :
     inlined, to ensure branches are decided at compilation time */
 LZ4_FORCE_INLINE int LZ4_compress_generic(
@@ -539,18 +562,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 {
     const BYTE* ip = (const BYTE*) source;
 
-    /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
-     * therefore safe to use no matter what mode we're in. Otherwise, we figure
-     * out if it's safe to leave as is or whether it needs to be reset.
-     */
-    const int resetTable = cctx->tableType != unusedTable && (
-                           cctx->tableType != tableType ||
-                            (tableType == byU16 &&
-                             cctx->currentOffset + inputSize >= 0xFFFFU) ||
-                           tableType == byPtr);
-
-    size_t currentOffset = ((tableType == byU32 || tableType == byU16) &&
-                            !resetTable) ? cctx->currentOffset : 1;
+    size_t currentOffset = cctx->currentOffset;
     const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
 
@@ -604,12 +616,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     }
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
 
-    if (resetTable) {
-      DEBUGLOG(4, "Resetting table in %p", cctx);
-      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 1;
-    }
-
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* First Byte */
@@ -819,15 +825,25 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     ctx->dictCtx = NULL;
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
-        if (inputSize < LZ4_64Klimit)
-            return LZ4_compress_generic(ctx, source, dest, inputSize,             0,    notLimited,                        byU16, noDict, noDictIssue, acceleration);
-        else
-            return LZ4_compress_generic(ctx, source, dest, inputSize,             0,    notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
+        if (inputSize < LZ4_64Klimit) {
+            const tableType_t tableType = byU16;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+        }
     } else {
-        if (inputSize < LZ4_64Klimit)
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput,                        byU16, noDict, noDictIssue, acceleration);
-        else
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
+        if (inputSize < LZ4_64Klimit) {
+            const tableType_t tableType = byU16;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+        }
     }
 }
 
@@ -842,7 +858,8 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
 #endif
-    LZ4_resetStream(ctxPtr);
+    ctxPtr->internal_donotuse.initCheck = 0;
+    ctxPtr->internal_donotuse.tableType = byPtr; /* always triggers a reset */
     result = LZ4_compress_fast_safeExtState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
 
 #if (LZ4_HEAPMODE)
@@ -1154,6 +1171,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
 
 int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
+    const tableType_t tableType = byU32;
     LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
     const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
@@ -1173,27 +1191,29 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
         }
     }
 
+    LZ4_resetTable(streamPtr, inputSize, tableType);
+
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
-            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
-            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration);
     }
 
     /* external dictionary mode */
     {   int result;
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
             if (streamPtr->dictCtx) {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, dictSmall, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
             } else {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             }
         } else {
             if (streamPtr->dictCtx) {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             } else {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             }
         }
         streamPtr->dictionary = (const BYTE*)source;
-- 
cgit v0.12


From 14ce912b705db03bd1d723456dd1a3242b7f7ea7 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Feb 2018 13:46:36 -0800
Subject: Switch Current Offset to 1 Only When in External Dictionary Context
 Mode

---
 lib/lz4.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 1f0450e..cace35f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -526,7 +526,8 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
 LZ4_FORCE_INLINE void LZ4_resetTable(
         LZ4_stream_t_internal* const cctx,
         const int inputSize,
-        const tableType_t tableType) {
+        const tableType_t tableType,
+        const dict_directive dictDirective) {
   /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
    * therefore safe to use no matter what mode we're in. Otherwise, we figure
    * out if it's safe to leave as is or whether it needs to be reset.
@@ -541,9 +542,12 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
   {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 1;
+      cctx->currentOffset = 0;
       cctx->tableType = unusedTable;
   }
+  if (dictDirective == usingExtDictCtx && cctx->currentOffset == 0) {
+      cctx->currentOffset = 1;
+  }
 }
 
 /** LZ4_compress_generic() :
@@ -827,21 +831,21 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
@@ -853,7 +857,6 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     int result;
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
-    ctxPtr->internal_donotuse.currentOffset = 1;
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1100,7 +1103,6 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-    LZ4_stream->internal_donotuse.currentOffset = 1;
     LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 
@@ -1191,10 +1193,10 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
         }
     }
 
-    LZ4_resetTable(streamPtr, inputSize, tableType);
 
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
+        LZ4_resetTable(streamPtr, inputSize, tableType, withPrefix64k);
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
             return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
@@ -1205,14 +1207,18 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
     {   int result;
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
             if (streamPtr->dictCtx) {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
             } else {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             }
         } else {
             if (streamPtr->dictCtx) {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             } else {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             }
         }
-- 
cgit v0.12


From 9dcd9abc14a33b6ac6c91dd727235db1daabe066 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Jan 2018 11:29:00 -0500
Subject: Make LZ4F_compressFrame_usingCDict Take a Compression Context

---
 lib/lz4frame.c        | 75 ++++++++++++++++++++++++++++++++++++---------------
 lib/lz4frame_static.h |  2 ++
 programs/lz4io.c      |  2 +-
 tests/framebench.c    |  2 ++
 tests/frametest.c     | 19 +++++++------
 5 files changed, 69 insertions(+), 31 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 9ff7766..e77fd35 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -47,6 +47,19 @@ You can contact the author at :
 
 
 /*-************************************
+*  Tuning parameters
+**************************************/
+/*
+ * LZ4_HEAPMODE :
+ * Select how default compression functions will allocate memory for their hash table,
+ * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
+ */
+#ifndef LZ4_HEAPMODE
+#  define LZ4_HEAPMODE 0
+#endif
+
+
+/*-************************************
 *  Memory routines
 **************************************/
 #include <stdlib.h>   /* malloc, calloc, free */
@@ -332,23 +345,18 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
  * @return : number of bytes written into dstBuffer,
  *           or an error code if it fails (can be tested using LZ4F_isError())
  */
-size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
+size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
+                                     void* dstBuffer, size_t dstCapacity,
                                const void* srcBuffer, size_t srcSize,
                                const LZ4F_CDict* cdict,
                                const LZ4F_preferences_t* preferencesPtr)
 {
-    LZ4F_cctx_t cctxI;
-    LZ4_stream_t lz4ctx;   /* pretty large on stack */
     LZ4F_preferences_t prefs;
     LZ4F_compressOptions_t options;
     BYTE* const dstStart = (BYTE*) dstBuffer;
     BYTE* dstPtr = dstStart;
     BYTE* const dstEnd = dstStart + dstCapacity;
 
-    memset(&cctxI, 0, sizeof(cctxI));
-    cctxI.version = LZ4F_VERSION;
-    cctxI.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
-
     if (preferencesPtr!=NULL)
         prefs = *preferencesPtr;
     else
@@ -361,33 +369,24 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
     if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
         prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* only one block => no need for inter-block link */
 
-    if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-        LZ4_resetStream(&lz4ctx);
-        cctxI.lz4CtxPtr = &lz4ctx;
-        cctxI.lz4CtxLevel = 2;
-    }  /* fast compression context pre-created on stack */
-
     memset(&options, 0, sizeof(options));
     options.stableSrc = 1;
 
     if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs))  /* condition to guarantee success */
         return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
 
-    { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs);  /* write header */
+    { size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs);  /* write header */
       if (LZ4F_isError(headerSize)) return headerSize;
       dstPtr += headerSize;   /* header size */ }
 
-    { size_t const cSize = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
+    { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
       if (LZ4F_isError(cSize)) return cSize;
       dstPtr += cSize; }
 
-    { size_t const tailSize = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options);   /* flush last block, and generate suffix */
+    { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, &options);   /* flush last block, and generate suffix */
       if (LZ4F_isError(tailSize)) return tailSize;
       dstPtr += tailSize; }
 
-    if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN)  /* Ctx allocation only for lz4hc */
-        FREEMEM(cctxI.lz4CtxPtr);
-
     return (dstPtr - dstStart);
 }
 
@@ -403,9 +402,41 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
                     const void* srcBuffer, size_t srcSize,
                     const LZ4F_preferences_t* preferencesPtr)
 {
-    return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity,
-                                         srcBuffer, srcSize,
-                                         NULL, preferencesPtr);
+    size_t result;
+#if (LZ4_HEAPMODE)
+    LZ4F_cctx_t *cctxPtr;
+    LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
+#else
+    LZ4F_cctx_t cctx;
+    LZ4_stream_t lz4ctx;
+    LZ4F_cctx_t *cctxPtr = &cctx;
+
+    MEM_INIT(&cctx, 0, sizeof(cctx));
+    cctx.version = LZ4F_VERSION;
+    cctx.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
+    if (preferencesPtr == NULL ||
+        preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN
+    ) {
+        LZ4_resetStream(&lz4ctx);
+        cctxPtr->lz4CtxPtr = &lz4ctx;
+        cctxPtr->lz4CtxLevel = 2;
+    }
+#endif
+
+    result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity,
+                                           srcBuffer, srcSize,
+                                           NULL, preferencesPtr);
+
+#if (LZ4_HEAPMODE)
+    LZ4F_freeCompressionContext(cctxPtr);
+#else
+    if (preferencesPtr != NULL &&
+        preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN
+    ) {
+        FREEMEM(cctxPtr->lz4CtxPtr);
+    }
+#endif
+    return result;
 }
 
 
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index a59b94b..be587e6 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -107,6 +107,7 @@ LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
 
 /*! LZ4_compressFrame_usingCDict() :
  *  Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+ *  cctx must point to a context created by LZ4F_createCompressionContext().
  *  If cdict==NULL, compress without a dictionary.
  *  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
  *  If this condition is not respected, function will fail (@return an errorCode).
@@ -115,6 +116,7 @@ LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
  * @return : number of bytes written into dstBuffer.
  *           or an error code if it fails (can be tested using LZ4F_isError()) */
 LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
+    LZ4F_cctx* cctx,
     void* dst, size_t dstCapacity,
     const void* src, size_t srcSize,
     const LZ4F_CDict* cdict,
diff --git a/programs/lz4io.c b/programs/lz4io.c
index c712fe1..ca13316 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -560,7 +560,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
     /* single-block file */
     if (readSize < blockSize) {
         /* Compress in single pass */
-        size_t cSize = LZ4F_compressFrame_usingCDict(dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
+        size_t cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
         if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
         compressedfilesize = cSize;
         DISPLAYUPDATE(2, "\rRead : %u MB   ==> %.2f%%   ",
diff --git a/tests/framebench.c b/tests/framebench.c
index 21c3704..a7a270b 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -30,6 +30,7 @@ typedef struct {
 
 size_t compress_frame(bench_params_t *p) {
   size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
   const char* ibuf = p->ibuf;
@@ -43,6 +44,7 @@ size_t compress_frame(bench_params_t *p) {
   prefs->frameInfo.contentSize = isize;
 
   oused = LZ4F_compressFrame_usingCDict(
+    cctx,
     obuf,
     osize,
     ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
diff --git a/tests/frametest.c b/tests/frametest.c
index 986a16d..74d9c88 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -164,7 +164,7 @@ static unsigned FUZ_highbit(U32 v32)
 /*-*******************************************************
 *  Tests
 *********************************************************/
-#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) goto _output_error
+#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) { fprintf(stderr, "%s\n", LZ4F_getErrorName(v)); goto _output_error; }
 #define CHECK(f)   { LZ4F_errorCode_t const CHECK_V(err_ , f); }
 
 int basicTests(U32 seed, double compressibility)
@@ -509,23 +509,25 @@ int basicTests(U32 seed, double compressibility)
         CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
     }
 
-
     /* Dictionary compression test */
     {   size_t const dictSize = 63 KB;
         size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL);
         size_t cSizeNoDict, cSizeWithDict;
         LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
         if (cdict == NULL) goto _output_error;
+        CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
         CHECK_V(cSizeNoDict,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               NULL, NULL) );
         DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
 
+        CHECK( LZ4F_freeCompressionContext(cctx) );
+        CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
         CHECK_V(cSizeWithDict,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               cdict, NULL) );
         DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
@@ -557,7 +559,7 @@ int basicTests(U32 seed, double compressibility)
             memset(&cParams, 0, sizeof(cParams));
             cParams.compressionLevel = -3;
             CHECK_V(cSizeLevelMax,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
@@ -569,7 +571,7 @@ int basicTests(U32 seed, double compressibility)
             memset(&cParams, 0, sizeof(cParams));
             cParams.compressionLevel = LZ4F_compressionLevel_max();
             CHECK_V(cSizeLevelMax,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
@@ -584,7 +586,7 @@ int basicTests(U32 seed, double compressibility)
             cParams.frameInfo.blockMode = LZ4F_blockLinked;
             cParams.frameInfo.blockSizeID = LZ4F_max64KB;
             CHECK_V(cSizeContiguous,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
                                               CNBuffer, inSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
@@ -620,7 +622,7 @@ int basicTests(U32 seed, double compressibility)
             cParams.frameInfo.blockMode = LZ4F_blockIndependent;
             cParams.frameInfo.blockSizeID = LZ4F_max64KB;
             CHECK_V(cSizeIndep,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
                                               CNBuffer, inSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
@@ -647,6 +649,7 @@ int basicTests(U32 seed, double compressibility)
         }
 
         LZ4F_freeCDict(cdict);
+        CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
     }
 
 
-- 
cgit v0.12


From 80790c587b55b8e0b932b64c892ba984cde1b70f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Jan 2018 12:06:43 -0500
Subject: Copy the Dict Table Into the Context for Large Compressions

---
 lib/lz4.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index cace35f..f7de564 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1205,20 +1205,25 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* external dictionary mode */
     {   int result;
-        if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-            if (streamPtr->dictCtx) {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
+        if (streamPtr->dictCtx && inputSize < 2 KB) {
+            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
+            if ((streamPtr->dictCtx->dictSize < 64 KB) && (streamPtr->dictCtx->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
             } else {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             }
         } else {
             if (streamPtr->dictCtx) {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
+              /* For compressing large blobs, it is faster to pay the setup
+               * cost to copy the dictionary's tables into the active context,
+               * so that the compression loop is only looking in one table.
+               */
+              memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
+            }
+            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
+            if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             } else {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             }
         }
-- 
cgit v0.12


From efc419a6d4448a5806715e967384ea85f880fe59 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Jan 2018 17:29:50 -0500
Subject: Replace calloc() Calls With malloc() Where Possible

---
 lib/lz4.c      | 12 ++++++------
 lib/lz4frame.c | 31 ++++++++++++++++---------------
 lib/lz4hc.c    |  2 +-
 3 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index f7de564..990096c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -154,7 +154,8 @@
 *  Memory routines
 **************************************/
 #include <stdlib.h>   /* malloc, calloc, free */
-#define ALLOCATOR(n,s) calloc(n,s)
+#define ALLOC(s) malloc(s)
+#define ALLOC_AND_ZERO(s) calloc(1,s)
 #define FREEMEM        free
 #include <string.h>   /* memset, memcpy */
 #define MEM_INIT       memset
@@ -856,7 +857,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
 {
     int result;
 #if (LZ4_HEAPMODE)
-    LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1070,7 +1071,7 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
 int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)
 {
 #if (LZ4_HEAPMODE)
-    LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
 #else
     LZ4_stream_t ctxBody;
     LZ4_stream_t* ctx = &ctxBody;
@@ -1092,7 +1093,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
 
 LZ4_stream_t* LZ4_createStream(void)
 {
-    LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64);
+    LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));
     LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal));    /* A compilation error here means LZ4_STREAMSIZE is not large enough */
     DEBUGLOG(4, "LZ4_createStream %p", lz4s);
     LZ4_resetStream(lz4s);
@@ -1193,7 +1194,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
         }
     }
 
-
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
         LZ4_resetTable(streamPtr, inputSize, tableType, withPrefix64k);
@@ -1493,7 +1493,7 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
 
 LZ4_streamDecode_t* LZ4_createStreamDecode(void)
 {
-    LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t));
+    LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t));
     return lz4s;
 }
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index e77fd35..8ed89e3 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -63,7 +63,8 @@ You can contact the author at :
 *  Memory routines
 **************************************/
 #include <stdlib.h>   /* malloc, calloc, free */
-#define ALLOCATOR(s)   calloc(1,s)
+#define ALLOC(s)   malloc(s)
+#define ALLOC_AND_ZERO(s)   calloc(1,s)
 #define FREEMEM        free
 #include <string.h>   /* memset, memcpy, memmove */
 #define MEM_INIT       memset
@@ -300,7 +301,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
                                           size_t alreadyBuffered)
 {
     LZ4F_preferences_t prefsNull;
-    memset(&prefsNull, 0, sizeof(prefsNull));
+    MEM_INIT(&prefsNull, 0, sizeof(prefsNull));
     prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;   /* worst case */
     {   const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
         U32 const flush = prefsPtr->autoFlush | (srcSize==0);
@@ -329,7 +330,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
     size_t const headerSize = maxFHSize;      /* max header size, including optional fields */
 
     if (preferencesPtr!=NULL) prefs = *preferencesPtr;
-    else memset(&prefs, 0, sizeof(prefs));
+    else MEM_INIT(&prefs, 0, sizeof(prefs));
     prefs.autoFlush = 1;
 
     return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);;
@@ -360,7 +361,7 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
     if (preferencesPtr!=NULL)
         prefs = *preferencesPtr;
     else
-        memset(&prefs, 0, sizeof(prefs));
+        MEM_INIT(&prefs, 0, sizeof(prefs));
     if (prefs.frameInfo.contentSize != 0)
         prefs.frameInfo.contentSize = (U64)srcSize;   /* auto-correct content size if selected (!=0) */
 
@@ -369,7 +370,7 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
     if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
         prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* only one block => no need for inter-block link */
 
-    memset(&options, 0, sizeof(options));
+    MEM_INIT(&options, 0, sizeof(options));
     options.stableSrc = 1;
 
     if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs))  /* condition to guarantee success */
@@ -465,7 +466,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
         dictStart += dictSize - 64 KB;
         dictSize = 64 KB;
     }
-    cdict->dictContent = ALLOCATOR(dictSize);
+    cdict->dictContent = ALLOC(dictSize);
     cdict->fastCtx = LZ4_createStream();
     cdict->HCCtx = LZ4_createStreamHC();
     if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
@@ -504,7 +505,7 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict)
  */
 LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
 {
-    LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t));
+    LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t));
     if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
 
     cctxPtr->version = version;
@@ -547,7 +548,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     BYTE* headerStart;
 
     if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
-    memset(&prefNull, 0, sizeof(prefNull));
+    MEM_INIT(&prefNull, 0, sizeof(prefNull));
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
     cctxPtr->prefs = *preferencesPtr;
 
@@ -588,7 +589,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         if (cctxPtr->maxBufferSize < requiredBuffSize) {
             cctxPtr->maxBufferSize = 0;
             FREEMEM(cctxPtr->tmpBuff);
-            cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
+            cctxPtr->tmpBuff = (BYTE*)ALLOC(requiredBuffSize);
             if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             cctxPtr->maxBufferSize = requiredBuffSize;
     }   }
@@ -813,7 +814,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
         return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
-    memset(&cOptionsNull, 0, sizeof(cOptionsNull));
+    MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull));
     if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
 
     /* complete tmp buffer */
@@ -1017,7 +1018,7 @@ struct LZ4F_dctx_s {
  */
 LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
 {
-    LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
+    LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx));
     if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC);
 
     dctx->version = versionNumber;
@@ -1089,7 +1090,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
 
     /* need to decode header to get frameInfo */
     if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete);   /* minimal frame header size */
-    memset(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
+    MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
 
     /* special case : skippable frames */
     if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) {
@@ -1316,7 +1317,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
     size_t nextSrcSizeHint = 1;
 
 
-    memset(&optionsNull, 0, sizeof(optionsNull));
+    MEM_INIT(&optionsNull, 0, sizeof(optionsNull));
     if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull;
     *srcSizePtr = 0;
     *dstSizePtr = 0;
@@ -1365,11 +1366,11 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 if (bufferNeeded > dctx->maxBufferSize) {   /* tmp buffers too small */
                     dctx->maxBufferSize = 0;   /* ensure allocation will be re-attempted on next entry*/
                     FREEMEM(dctx->tmpIn);
-                    dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize + 4 /* block checksum */);
+                    dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + 4 /* block checksum */);
                     if (dctx->tmpIn == NULL)
                         return err0r(LZ4F_ERROR_allocation_failed);
                     FREEMEM(dctx->tmpOutBuffer);
-                    dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
+                    dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded);
                     if (dctx->tmpOutBuffer== NULL)
                         return err0r(LZ4F_ERROR_allocation_failed);
                     dctx->maxBufferSize = bufferNeeded;
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 2a6e080..0c1b20a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -858,7 +858,7 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
 
 void* LZ4_createHC (char* inputBuffer)
 {
-    LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t));
+    LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
     if (hc4 == NULL) return NULL;   /* not enough memory */
     LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
     hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer;
-- 
cgit v0.12


From d6ed9a7799c35d9d259af36c3142739216ba8a58 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 30 Jan 2018 15:22:29 -0500
Subject: Avoid dictionary == NULL Check

---
 lib/lz4.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 990096c..3e456ac 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -590,6 +590,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         (const BYTE*) source - dictCtx->currentOffset :
         (const BYTE*) source - dictSize - currentOffset;
     const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
+    const BYTE* dictLowLimit;
 
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
@@ -619,6 +620,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         lowLimit = (const BYTE*)source;
         break;
     }
+
+    dictLowLimit = dictionary ? dictionary : lowLimit;
+
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
 
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
@@ -653,15 +657,15 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                         /* TODO: use precalc-ed hash? */
                         match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                         refDelta = dictDelta;
-                        lowLimit = dictionary;
+                        lowLimit = dictLowLimit;
                     } else {
                         refDelta = 0;
                         lowLimit = (const BYTE*)source;
                     }
                 } else if (dictDirective==usingExtDict) {
-                    if (match < (const BYTE*)source && dictionary != NULL) {
+                    if (match < (const BYTE*)source) {
                         refDelta = dictDelta;
-                        lowLimit = dictionary;
+                        lowLimit = dictLowLimit;
                     } else {
                         refDelta = 0;
                         lowLimit = (const BYTE*)source;
@@ -703,7 +707,7 @@ _next_match:
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictionary)) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictLowLimit)) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
@@ -754,15 +758,15 @@ _next_match:
                 /* TODO: use precalc-ed hash? */
                 match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                 refDelta = dictDelta;
-                lowLimit = dictionary;
+                lowLimit = dictLowLimit;
             } else {
                 refDelta = 0;
                 lowLimit = (const BYTE*)source;
             }
         } else if (dictDirective==usingExtDict) {
-            if (match < (const BYTE*)source && dictionary != NULL) {
+            if (match < (const BYTE*)source) {
                 refDelta = dictDelta;
-                lowLimit = dictionary;
+                lowLimit = dictLowLimit;
             } else {
                 refDelta = 0;
                 lowLimit = (const BYTE*)source;
-- 
cgit v0.12


From 1c4601d64377cdd4d5fc17ccd1f6e1426c7ddd04 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 16 Feb 2018 17:33:51 -0800
Subject: Restore DictIssue Check

---
 lib/lz4.c | 37 +++++++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 3e456ac..a401ee2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -599,9 +599,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     U32 forwardH;
 
-    /* TODO: resurrect dictIssue check */
-    (void)dictIssue;
-
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     switch(dictDirective)
@@ -673,7 +670,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
 
-            } while ( match < lowRefLimit
+            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
                 || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
         }
@@ -772,7 +769,7 @@ _next_match:
                 lowLimit = (const BYTE*)source;
         }   }
         LZ4_putPosition(ip, cctx->hashTable, tableType, base);
-        if ( match >= lowRefLimit
+        if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
             && (match+MAX_DISTANCE>=ip)
             && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
         { token=op++; *token=0; goto _next_match; }
@@ -837,21 +834,37 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            }
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            }
         }
     }
 }
@@ -1249,7 +1262,11 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
     LZ4_renormDictT(streamPtr, smallest);
 
-    result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+    if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
+      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+    } else {
+      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+    }
 
     streamPtr->dictionary = (const BYTE*)source;
     streamPtr->dictSize = (U32)inputSize;
-- 
cgit v0.12


From d571d0cdbaa7032fff48543f272f6d5472041aa7 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 5 Mar 2018 11:59:22 -0500
Subject: Avoid DictSmall Checks By Strategically Bumping CurrentOffset

---
 lib/lz4.c | 45 +++++++++++++++++++++++----------------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index a401ee2..7c5b444 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -546,7 +546,10 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
       cctx->currentOffset = 0;
       cctx->tableType = unusedTable;
   }
-  if (dictDirective == usingExtDictCtx && cctx->currentOffset == 0) {
+  if (dictDirective == usingExtDictCtx &&
+      tableType != byPtr &&
+      cctx->currentOffset == 0)
+  {
       cctx->currentOffset = 1;
   }
 }
@@ -842,11 +845,8 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
-            } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
-            }
+            ctx->currentOffset += 64 KB;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
@@ -860,11 +860,8 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
-            } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
-            }
+            ctx->currentOffset += 64 KB;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
 }
@@ -1222,21 +1219,25 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* external dictionary mode */
     {   int result;
-        if (streamPtr->dictCtx && inputSize < 2 KB) {
-            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
-            if ((streamPtr->dictCtx->dictSize < 64 KB) && (streamPtr->dictCtx->dictSize < streamPtr->currentOffset)) {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
+        if (streamPtr->dictCtx) {
+            /* We depend here on the fact that dictCtx'es (produced by
+             * LZ4_loadDict) guarantee that their tables contain no references
+             * to offsets between dictCtx->currentOffset - 64 KB and
+             * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe
+             * to use noDictIssue even when the dict isn't a full 64 KB.
+             */
+            if (inputSize > 2 KB) {
+                /* For compressing large blobs, it is faster to pay the setup
+                 * cost to copy the dictionary's tables into the active context,
+                 * so that the compression loop is only looking in one table.
+                 */
+                memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             }
         } else {
-            if (streamPtr->dictCtx) {
-              /* For compressing large blobs, it is faster to pay the setup
-               * cost to copy the dictionary's tables into the active context,
-               * so that the compression loop is only looking in one table.
-               */
-              memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
-            }
             LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
-- 
cgit v0.12


From 00eadadbfc37587e88733942a252c2f106c67ef2 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Feb 2018 17:06:24 -0800
Subject: Reset Table on Inputs Larger than 2KB

---
 lib/lz4.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 7c5b444..4810910 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -539,7 +539,8 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
        cctx->currentOffset + inputSize >= 0xFFFFU) ||
       (tableType == byU32 &&
        cctx->currentOffset > 1 GB) ||
-      tableType == byPtr))
+      tableType == byPtr ||
+      inputSize >= 2 KB))
   {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-- 
cgit v0.12


From b78cf67c9624a6a826ceaa2a9d65f5feb1dd0cfd Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 6 Mar 2018 11:52:02 -0500
Subject: Move to 4KB Cut-Off

---
 lib/lz4.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4810910..d147681 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -540,7 +540,7 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
       (tableType == byU32 &&
        cctx->currentOffset > 1 GB) ||
       tableType == byPtr ||
-      inputSize >= 2 KB))
+      inputSize >= 4 KB))
   {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
@@ -1227,7 +1227,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
              * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe
              * to use noDictIssue even when the dict isn't a full 64 KB.
              */
-            if (inputSize > 2 KB) {
+            if (inputSize > 4 KB) {
                 /* For compressing large blobs, it is faster to pay the setup
                  * cost to copy the dictionary's tables into the active context,
                  * so that the compression loop is only looking in one table.
-- 
cgit v0.12


From 64bcbf400ea6c955b029f74e1214dcd03b90f934 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 6 Mar 2018 15:53:22 -0500
Subject: Optimize Dict Check Condition

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d147681..4f98efa 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -708,7 +708,7 @@ _next_match:
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictLowLimit)) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && ((lowLimit==dictLowLimit) & (dictionary != NULL))) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
-- 
cgit v0.12


From b4335a6585fe55aa81bcadaa29e265ffe9550ff2 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 8 Mar 2018 12:29:45 -0500
Subject: Further Avoid a dictionary==NULL Check

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4f98efa..da68ddd 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -708,7 +708,7 @@ _next_match:
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && ((lowLimit==dictLowLimit) & (dictionary != NULL))) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && lowLimit==dictionary) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
-- 
cgit v0.12


From 6716325ae8db3f177630c875622401e35e4a84e5 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 8 Mar 2018 12:30:34 -0500
Subject: Remove Switch In Favor of Ternary Statement

---
 lib/lz4.c | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index da68ddd..809faf9 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -605,27 +605,11 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
-    switch(dictDirective)
-    {
-    case noDict:
-    default:
-        lowLimit = (const BYTE*)source;
-        break;
-    case withPrefix64k:
-        lowLimit = (const BYTE*)source - dictSize;
-        break;
-    case usingExtDict:
-        lowLimit = (const BYTE*)source;
-        break;
-    case usingExtDictCtx:
-        lowLimit = (const BYTE*)source;
-        break;
-    }
 
+    lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
     dictLowLimit = dictionary ? dictionary : lowLimit;
 
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
-
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* First Byte */
-- 
cgit v0.12


From 5f8967b98364629310d0013b356070ab445f9e66 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 9 Mar 2018 12:14:42 -0500
Subject: Specialize _extState() for Clean Ctx Rather Than Calling
 _safeExtState()

---
 lib/lz4.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 809faf9..82c0ae3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -805,8 +805,24 @@ _clean_up:
 
 int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
-  LZ4_resetStream((LZ4_stream_t*)state);
-  return LZ4_compress_fast_safeExtState(state, source, dest, inputSize, maxOutputSize, acceleration);
+    LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
+    if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
+    LZ4_resetStream((LZ4_stream_t*)state);
+    if (maxOutputSize >= LZ4_compressBound(inputSize)) {
+        if (inputSize < LZ4_64Klimit) {
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, byU16, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+        }
+    } else {
+        if (inputSize < LZ4_64Klimit) {;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+        }
+    }
 }
 
 
@@ -861,9 +877,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
 #endif
-    ctxPtr->internal_donotuse.initCheck = 0;
-    ctxPtr->internal_donotuse.tableType = byPtr; /* always triggers a reset */
-    result = LZ4_compress_fast_safeExtState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
+    result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
 
 #if (LZ4_HEAPMODE)
     FREEMEM(ctxPtr);
-- 
cgit v0.12


From e34716cc3f54d3fca1cbd797e8af003cd3a63bc8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 8 Mar 2018 14:09:06 -0500
Subject: Preserve currentOffset==0 When Possible

---
 lib/lz4.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 82c0ae3..1ea8ba1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -846,7 +846,9 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            ctx->currentOffset += 64 KB;
+            if (ctx->currentOffset) {
+              ctx->currentOffset += 64 KB;
+            }
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
@@ -861,7 +863,9 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            ctx->currentOffset += 64 KB;
+            if (ctx->currentOffset) {
+              ctx->currentOffset += 64 KB;
+            }
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
-- 
cgit v0.12


From 3ecc1d7a5b0f911c57851cc8a925d7be2518d358 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 16:11:44 -0400
Subject: Minor Style Fixes

---
 lib/lz4frame.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 8ed89e3..16769fe 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -50,12 +50,12 @@ You can contact the author at :
 *  Tuning parameters
 **************************************/
 /*
- * LZ4_HEAPMODE :
+ * LZ4F_HEAPMODE :
  * Select how default compression functions will allocate memory for their hash table,
  * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
  */
-#ifndef LZ4_HEAPMODE
-#  define LZ4_HEAPMODE 0
+#ifndef LZ4F_HEAPMODE
+#  define LZ4F_HEAPMODE 0
 #endif
 
 
@@ -404,7 +404,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
                     const LZ4F_preferences_t* preferencesPtr)
 {
     size_t result;
-#if (LZ4_HEAPMODE)
+#if (LZ4F_HEAPMODE)
     LZ4F_cctx_t *cctxPtr;
     LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
 #else
@@ -416,8 +416,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     cctx.version = LZ4F_VERSION;
     cctx.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
     if (preferencesPtr == NULL ||
-        preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN
-    ) {
+        preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN)
+    {
         LZ4_resetStream(&lz4ctx);
         cctxPtr->lz4CtxPtr = &lz4ctx;
         cctxPtr->lz4CtxLevel = 2;
@@ -428,12 +428,12 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
                                            srcBuffer, srcSize,
                                            NULL, preferencesPtr);
 
-#if (LZ4_HEAPMODE)
+#if (LZ4F_HEAPMODE)
     LZ4F_freeCompressionContext(cctxPtr);
 #else
     if (preferencesPtr != NULL &&
-        preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN
-    ) {
+        preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN)
+    {
         FREEMEM(cctxPtr->lz4CtxPtr);
     }
 #endif
-- 
cgit v0.12


From 1df5d911aa23e645b873164382a40186ed704bac Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 16:11:55 -0400
Subject: Hoist LZ4F Dictionary Setup into Helper LZ4F_applyCDict()

---
 lib/lz4frame.c | 72 ++++++++++++++++++++--------------------------------------
 1 file changed, 25 insertions(+), 47 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 16769fe..4cc2ef3 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -531,6 +531,28 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 }
 
 
+static void LZ4F_applyCDict(void* ctx,
+                            const LZ4F_CDict* cdict,
+                            int level) {
+    if (level < LZ4HC_CLEVEL_MIN) {
+        LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
+        assert(!internal_ctx->initCheck);
+        /* Clear any local dictionary */
+        internal_ctx->dictionary = NULL;
+        internal_ctx->dictSize = 0;
+        /* Point to the dictionary context */
+        internal_ctx->dictCtx = cdict ? &(cdict->fastCtx->internal_donotuse) : NULL;
+    } else {
+        if (cdict) {
+            memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
+            LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
+        } else {
+            LZ4_resetStreamHC((LZ4_streamHC_t*)(ctx), level);
+        }
+    }
+}
+
+
 /*! LZ4F_compressBegin_usingCDict() :
  *  init streaming compression and writes frame header into dstBuffer.
  *  dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
@@ -601,39 +623,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->cdict = cdict;
     if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
         /* frame init only for blockLinked : blockIndependent will be init at each block */
-        if (cdict) {
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
-                assert(!internal_ctx->initCheck);
-                if (internal_ctx->currentOffset > 1 GB) {
-                    /* Init the context */
-                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
-                }
-                /* Clear any local dictionary */
-                internal_ctx->dictionary = NULL;
-                internal_ctx->dictSize = 0;
-                /* Point to the dictionary context */
-                internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
-            } else {
-                memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
-                LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
-            }
-        } else {
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
-                assert(!internal_ctx->initCheck);
-                if (internal_ctx->currentOffset > 1 GB) {
-                    /* Init the context */
-                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
-                }
-                /* Clear any local dictionary */
-                internal_ctx->dictionary = NULL;
-                internal_ctx->dictSize = 0;
-                internal_ctx->dictCtx = NULL;
-            } else {
-                LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
-            }
-        }
+        LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
     }
 
     /* Magic Number */
@@ -728,21 +718,10 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
-    LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)ctx)->internal_donotuse;
-    assert(!internal_ctx->initCheck);
-    if (internal_ctx->currentOffset > 1 GB) {
-        /* Init the context */
-        LZ4_resetStream((LZ4_stream_t*)ctx);
-    }
-    /* Clear any local dictionary */
-    internal_ctx->dictionary = NULL;
-    internal_ctx->dictSize = 0;
+    LZ4F_applyCDict(ctx, cdict, level);
     if (cdict) {
-        /* Point to the dictionary context */
-        internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
-        internal_ctx->dictCtx = NULL;
         return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
 }
@@ -757,8 +736,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     if (cdict) {
-        memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
-        LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
+        LZ4F_applyCDict(ctx, cdict, level);
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }
     return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
-- 
cgit v0.12


From f9fef255a12470b7beeaa8635b65fc837d835c84 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 18:13:24 -0400
Subject: Renames and Comment Fixes

---
 lib/lz4.c          | 32 ++++++++++++++++++++++----------
 lib/lz4.h          | 14 +++++++-------
 lib/lz4frame.c     |  2 +-
 tests/framebench.c |  2 +-
 4 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 1ea8ba1..94248be 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -524,7 +524,7 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
 }
 
 
-LZ4_FORCE_INLINE void LZ4_resetTable(
+LZ4_FORCE_INLINE void LZ4_prepareTable(
         LZ4_stream_t_internal* const cctx,
         const int inputSize,
         const tableType_t tableType,
@@ -547,6 +547,11 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
       cctx->currentOffset = 0;
       cctx->tableType = unusedTable;
   }
+  /* If the current offset is zero, we will never look in the external
+   * dictionary context, since there is no value a table entry can take that
+   * indicates a miss. In that case, we need to bump the offset to something
+   * non-zero.
+   */
   if (dictDirective == usingExtDictCtx &&
       tableType != byPtr &&
       cctx->currentOffset == 0)
@@ -825,8 +830,15 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
     }
 }
 
-
-int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+/**
+ * LZ4_compress_fast_extState_noReset is a variant of LZ4_compress_fast_extState
+ * that can be used when the state is known to have already been initialized
+ * (via LZ4_resetStream or an earlier call to LZ4_compress_fast_extState /
+ * LZ4_compress_fast_extState_noReset). This can provide significantly better
+ * performance when the context reset would otherwise be a significant part of
+ * the cost of the compression, e.g., when the data to be compressed is small.
+ */
+int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -837,7 +849,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -845,7 +857,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               ctx->currentOffset += 64 KB;
             }
@@ -854,7 +866,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -862,7 +874,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               ctx->currentOffset += 64 KB;
             }
@@ -1213,7 +1225,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
-        LZ4_resetTable(streamPtr, inputSize, tableType, withPrefix64k);
+        LZ4_prepareTable(streamPtr, inputSize, tableType, withPrefix64k);
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
             return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
@@ -1237,11 +1249,11 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
+                LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             }
         } else {
-            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
+            LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             } else {
diff --git a/lib/lz4.h b/lib/lz4.h
index 58a1e39..ca9d552 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -179,21 +179,21 @@ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int d
 
 
 /*!
-LZ4_compress_fast_safeExtState() :
+LZ4_compress_fast_extState_noReset() :
 LZ4_compress_fast_extState() :
     Same compression function, just using an externally allocated memory space to store compression state.
     Use LZ4_sizeofState() to know how much memory must be allocated,
     and allocate it on 8-bytes boundaries (using malloc() typically).
     Then, provide it as 'void* state' to compression function.
 
-    Use _safeExtState variant if LZ4_resetStream() was called on the state
-    buffer before being used for the first time (calls to this function leave
-    the state in a safe state, so zeroing is not required between calls).
-    Otherwise, using legacy _extState requires LZ4 to reinitialize the state
-    internally for every call.
+    Use the _noReset variant if LZ4_resetStream() was called on the state
+    buffer before being used for the first time (calls to both extState
+    functions leave the state in a safe state, so zeroing is not required
+    between calls). Otherwise, using the legacy _extState requires LZ4 to
+    reinitialize the state internally for every call.
 */
 LZ4LIB_API int LZ4_sizeofState(void);
-LZ4LIB_API int LZ4_compress_fast_safeExtState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 4cc2ef3..7acd7cf 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -722,7 +722,7 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize
     if (cdict) {
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
-        return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
+        return LZ4_compress_fast_extState_noReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
 }
 
diff --git a/tests/framebench.c b/tests/framebench.c
index a7a270b..fb4f38c 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -120,7 +120,7 @@ size_t compress_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_safeExtState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  oused = LZ4_compress_fast_extState_noReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
   obuf += oused;
 
   return obuf - p->obuf;
-- 
cgit v0.12


From 299f34909a2da7dc7096747e1c62cfe3994ec467 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 9 Mar 2018 12:05:31 -0500
Subject: Simpler Ternary Statements

---
 lib/lz4.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 94248be..6a680e6 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -582,9 +582,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
     const BYTE* const dictionary =
-        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictionary;
+        dictDirective == usingExtDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
-        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictSize;
+        dictDirective == usingExtDictCtx ? dictCtx->dictSize : cctx->dictSize;
 
     const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
     const BYTE* const dictEnd = dictionary + dictSize;
-- 
cgit v0.12


From 5149767a1b3caba4c17c0e79247c6092f7ae23ef Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 18:32:24 -0400
Subject: Add NULL Checks

---
 lib/lz4.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index 6a680e6..822d572 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -889,6 +889,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     int result;
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    if (ctxPtr == NULL) return 0;
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1101,6 +1102,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
 {
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    if (ctx == NULL) return 0;
 #else
     LZ4_stream_t ctxBody;
     LZ4_stream_t* ctx = &ctxBody;
@@ -1125,6 +1127,7 @@ LZ4_stream_t* LZ4_createStream(void)
     LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));
     LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal));    /* A compilation error here means LZ4_STREAMSIZE is not large enough */
     DEBUGLOG(4, "LZ4_createStream %p", lz4s);
+    if (lz4s == NULL) return NULL;
     LZ4_resetStream(lz4s);
     return lz4s;
 }
-- 
cgit v0.12


From b8e9c7785559b01f1f5ae0db6700e2673f4823db Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 18:46:54 -0400
Subject: Whitespace Fixes

---
 lib/lz4.c | 89 +++++++++++++++++++++++++++++++--------------------------------
 1 file changed, 44 insertions(+), 45 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 822d572..0209beb 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -529,35 +529,34 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
         const int inputSize,
         const tableType_t tableType,
         const dict_directive dictDirective) {
-  /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
-   * therefore safe to use no matter what mode we're in. Otherwise, we figure
-   * out if it's safe to leave as is or whether it needs to be reset.
-   */
-  if (cctx->tableType != unusedTable && (
-      cctx->tableType != tableType ||
-      (tableType == byU16 &&
-       cctx->currentOffset + inputSize >= 0xFFFFU) ||
-      (tableType == byU32 &&
-       cctx->currentOffset > 1 GB) ||
-      tableType == byPtr ||
-      inputSize >= 4 KB))
-  {
-      DEBUGLOG(4, "Resetting table in %p", cctx);
-      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 0;
-      cctx->tableType = unusedTable;
-  }
-  /* If the current offset is zero, we will never look in the external
-   * dictionary context, since there is no value a table entry can take that
-   * indicates a miss. In that case, we need to bump the offset to something
-   * non-zero.
-   */
-  if (dictDirective == usingExtDictCtx &&
-      tableType != byPtr &&
-      cctx->currentOffset == 0)
-  {
-      cctx->currentOffset = 1;
-  }
+    /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
+     * therefore safe to use no matter what mode we're in. Otherwise, we figure
+     * out if it's safe to leave as is or whether it needs to be reset.
+     */
+    if (cctx->tableType != unusedTable) {
+        if (cctx->tableType != tableType
+          || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU)
+          || (tableType == byU32 && cctx->currentOffset > 1 GB)
+          || tableType == byPtr
+          || inputSize >= 4 KB)
+        {
+            DEBUGLOG(4, "Resetting table in %p", cctx);
+            MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
+            cctx->currentOffset = 0;
+            cctx->tableType = unusedTable;
+        }
+    }
+    /* If the current offset is zero, we will never look in the external
+     * dictionary context, since there is no value a table entry can take that
+     * indicates a miss. In that case, we need to bump the offset to something
+     * non-zero.
+     */
+    if (dictDirective == usingExtDictCtx &&
+        tableType != byPtr &&
+        cctx->currentOffset == 0)
+    {
+        cctx->currentOffset = 1;
+    }
 }
 
 /** LZ4_compress_generic() :
@@ -793,12 +792,12 @@ _last_literals:
 
 _clean_up:
     if (dictDirective == usingExtDictCtx) {
-      /* Subsequent linked blocks can't use the dictionary. */
-      /* Instead, they use the block we just compressed. */
-      cctx->dictCtx = NULL;
-      cctx->dictSize = (U32)inputSize;
+        /* Subsequent linked blocks can't use the dictionary. */
+        /* Instead, they use the block we just compressed. */
+        cctx->dictCtx = NULL;
+        cctx->dictSize = (U32)inputSize;
     } else {
-      cctx->dictSize += (U32)inputSize;
+        cctx->dictSize += (U32)inputSize;
     }
     cctx->currentOffset += (U32)inputSize;
     cctx->tableType = tableType;
@@ -815,10 +814,10 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
     LZ4_resetStream((LZ4_stream_t*)state);
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, byU16, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {;
@@ -851,32 +850,32 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
             const tableType_t tableType = byU16;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              ctx->currentOffset += 64 KB;
+                ctx->currentOffset += 64 KB;
             }
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              ctx->currentOffset += 64 KB;
+                ctx->currentOffset += 64 KB;
             }
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
@@ -1282,9 +1281,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     LZ4_renormDictT(streamPtr, smallest);
 
     if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
     } else {
-      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
     }
 
     streamPtr->dictionary = (const BYTE*)source;
-- 
cgit v0.12


From 2be38a742917651359cdfc737b74bda8b50ece73 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 15:07:19 -0400
Subject: Rename Enums and Add Comment

---
 lib/lz4.c | 59 ++++++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 40 insertions(+), 19 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0209beb..f86d3ae 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -447,9 +447,32 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 *  Local Structures and types
 **************************************/
 typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
-typedef enum { unusedTable = 0, byPtr = 1, byU32 = 2, byU16 = 3 } tableType_t;
+typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t;
 
-typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingExtDictCtx } dict_directive;
+/**
+ * This enum distinguishes several different modes of accessing previous
+ * content in the stream.
+ *
+ * - noDict        : There is no preceding content.
+ * - withPrefix64k : Table entries up to ctx->dictSize before the current blob
+ *                   blob being compressed are valid and refer to the preceding
+ *                   content (of length ctx->dictSize), which is available
+ *                   contiguously preceding in memory the content currently
+ *                   being compressed.
+ * - usingExtDict  : Like withPrefix64k, but the preceding content is somewhere
+ *                   else in memory, starting at ctx->dictionary with length
+ *                   ctx->dictSize.
+ * - usingDictCtx  : Like usingExtDict, but everything concerning the preceding
+ *                   content is in a separate context, pointed to by
+ *                   ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table
+ *                   entries in the current context that refer to positions
+ *                   preceding the beginning of the current compression are
+ *                   ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx
+ *                   ->dictSize describe the location and size of the preceding
+ *                   content, and matches are found by looking in the ctx
+ *                   ->dictCtx->hashTable.
+ */
+typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive;
 typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
 
 typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
@@ -497,7 +520,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy
 {
     switch (tableType)
     {
-    case unusedTable: { /* illegal! */ assert(0); return; }
+    case clearedTable: { /* illegal! */ assert(0); return; }
     case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; }
     case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; }
     case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; }
@@ -533,7 +556,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * therefore safe to use no matter what mode we're in. Otherwise, we figure
      * out if it's safe to leave as is or whether it needs to be reset.
      */
-    if (cctx->tableType != unusedTable) {
+    if (cctx->tableType != clearedTable) {
         if (cctx->tableType != tableType
           || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU)
           || (tableType == byU32 && cctx->currentOffset > 1 GB)
@@ -543,7 +566,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
             DEBUGLOG(4, "Resetting table in %p", cctx);
             MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
             cctx->currentOffset = 0;
-            cctx->tableType = unusedTable;
+            cctx->tableType = clearedTable;
         }
     }
     /* If the current offset is zero, we will never look in the external
@@ -551,7 +574,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * indicates a miss. In that case, we need to bump the offset to something
      * non-zero.
      */
-    if (dictDirective == usingExtDictCtx &&
+    if (dictDirective == usingDictCtx &&
         tableType != byPtr &&
         cctx->currentOffset == 0)
     {
@@ -581,9 +604,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
     const BYTE* const dictionary =
-        dictDirective == usingExtDictCtx ? dictCtx->dictionary : cctx->dictionary;
+        dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
-        dictDirective == usingExtDictCtx ? dictCtx->dictSize : cctx->dictSize;
+        dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
 
     const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
     const BYTE* const dictEnd = dictionary + dictSize;
@@ -594,7 +617,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     /* the dictCtx currentOffset is indexed on the start of the dictionary,
      * while a dictionary in the current context precedes the currentOffset */
-    const BYTE* dictBase = dictDirective == usingExtDictCtx ?
+    const BYTE* dictBase = dictDirective == usingDictCtx ?
         (const BYTE*) source - dictCtx->currentOffset :
         (const BYTE*) source - dictSize - currentOffset;
     const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
@@ -640,10 +663,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 assert(ip < mflimitPlusOne);
 
                 match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
-                if (dictDirective == usingExtDictCtx) {
+                if (dictDirective == usingDictCtx) {
                     if (match < (const BYTE*)source) {
                         /* there was no match, try the dictionary */
-                        /* TODO: use precalc-ed hash? */
                         match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                         refDelta = dictDelta;
                         lowLimit = dictLowLimit;
@@ -696,7 +718,7 @@ _next_match:
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && lowLimit==dictionary) {
+            if ((dictDirective==usingExtDict || dictDirective==usingDictCtx) && lowLimit==dictionary) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
@@ -741,10 +763,9 @@ _next_match:
 
         /* Test next position */
         match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
-        if (dictDirective == usingExtDictCtx) {
+        if (dictDirective == usingDictCtx) {
             if (match < (const BYTE*)source) {
                 /* there was no match, try the dictionary */
-                /* TODO: use precalc-ed hash? */
                 match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                 refDelta = dictDelta;
                 lowLimit = dictLowLimit;
@@ -791,7 +812,7 @@ _last_literals:
     retval = (((char*)op)-dest);
 
 _clean_up:
-    if (dictDirective == usingExtDictCtx) {
+    if (dictDirective == usingDictCtx) {
         /* Subsequent linked blocks can't use the dictionary. */
         /* Instead, they use the block we just compressed. */
         cctx->dictCtx = NULL;
@@ -1135,7 +1156,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-    LZ4_stream->internal_donotuse.tableType = unusedTable;
+    LZ4_stream->internal_donotuse.tableType = clearedTable;
 }
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
@@ -1158,7 +1179,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
 
     if ((dict->initCheck)
-      || (dict->tableType != byU32 && dict->tableType != unusedTable)
+      || (dict->tableType != byU32 && dict->tableType != clearedTable)
       || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
         LZ4_resetStream(LZ4_dict);
 
@@ -1251,8 +1272,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDictCtx);
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
+                LZ4_prepareTable(streamPtr, inputSize, tableType, usingDictCtx);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
         } else {
             LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);
-- 
cgit v0.12


From c4aef7cd62cff0b71464dc58f40efc284bcb5a7b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 15:18:08 -0400
Subject: Restore checkTag Cleaning

---
 tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index 77d9e36..a351d35 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -113,7 +113,7 @@ clean:
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
         framebench$(EXT) \
-        fasttest$(EXT) datagen$(EXT)
+        fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
 
-- 
cgit v0.12


From 146e67653171b9b43322e2ab3d8be7b085b03bf7 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 15:42:03 -0400
Subject: Restore LZ4_sizeofStreamState, We Didn't Actually Need to Delete It

---
 lib/lz4.c | 3 +++
 lib/lz4.h | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index f86d3ae..48cdad1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1703,6 +1703,9 @@ They are only provided here for compatibility with older user programs.
 int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
 int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
 
+/* Obsolete Streaming functions */
+
+int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
 
 /* Obsolete streaming decompression functions */
 
diff --git a/lib/lz4.h b/lib/lz4.h
index ca9d552..e4a257b 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -477,6 +477,9 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
 LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
+/* Obsolete streaming functions; use new streaming interface whenever possible */
+LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
+
 /* Obsolete streaming decoding functions */
 LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
 LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
-- 
cgit v0.12


From 640db34e43481fdd7c69462ca8a84c9d964cdfd6 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 17:35:44 -0400
Subject: Another Allocation Fail Check

---
 lib/lz4frame.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 7acd7cf..d0c69d1 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -406,7 +406,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     size_t result;
 #if (LZ4F_HEAPMODE)
     LZ4F_cctx_t *cctxPtr;
-    LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
+    result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
+    if (LZ4F_isError(result)) return result;
 #else
     LZ4F_cctx_t cctx;
     LZ4_stream_t lz4ctx;
-- 
cgit v0.12


From 995756f218e4f144953ca4dfc8df37493e8f775b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 17:45:09 -0400
Subject: Split lz4CtxLevel into Two Fields

---
 lib/lz4frame.c | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d0c69d1..87e209f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -202,15 +202,8 @@ typedef struct LZ4F_cctx_s
     U64    totalInSize;
     XXH32_state_t xxh;
     void*  lz4CtxPtr;
-    /* lz4CtxLevel records both what size of memory has been allocated into the
-     * ctx pointer field and what it's being used as.
-     * - The low bit (& 1) value indicates whether the ctx is being used for
-     *   hc (1) or not (0).
-     * - The next bit (& 2) indicates whether the allocated memory is big
-     *   enough for a non-hc context.
-     * - The next bit (& 4) indicates whether the allocated memory is big
-     *   enough for an hc context. */
-    U32    lz4CtxLevel;
+    U16    lz4CtxAlloc; // sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx
+    U16    lz4CtxState; // in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx
 } LZ4F_cctx_t;
 
 
@@ -421,7 +414,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     {
         LZ4_resetStream(&lz4ctx);
         cctxPtr->lz4CtxPtr = &lz4ctx;
-        cctxPtr->lz4CtxLevel = 2;
+        cctxPtr->lz4CtxAlloc = 1;
+        cctxPtr->lz4CtxState = 1;
     }
 #endif
 
@@ -576,9 +570,8 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->prefs = *preferencesPtr;
 
     /* Ctx Management */
-    {   U32 const ctxSufficientAllocBits = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 2 : 6;
-        U32 const ctxTypeBit = cctxPtr->prefs.compressionLevel >= LZ4HC_CLEVEL_MIN;
-        if ((cctxPtr->lz4CtxLevel & ctxSufficientAllocBits) != ctxSufficientAllocBits) {
+    {   U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
+        if (cctxPtr->lz4CtxAlloc < ctxTypeID) {
             FREEMEM(cctxPtr->lz4CtxPtr);
             if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
@@ -586,17 +579,17 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
             }
             if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
-            cctxPtr->lz4CtxLevel = ctxSufficientAllocBits | ctxTypeBit;
-        } else if ((cctxPtr->lz4CtxLevel & 1) != ctxTypeBit) {
+            cctxPtr->lz4CtxAlloc = ctxTypeID;
+            cctxPtr->lz4CtxState = ctxTypeID;
+        } else if (cctxPtr->lz4CtxState != ctxTypeID) {
             /* otherwise, a sufficient buffer is allocated, but we need to
              * reset it to the correct context type */
             if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
                 LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
-                cctxPtr->lz4CtxLevel &= ~1;
             } else {
                 LZ4_resetStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
-                cctxPtr->lz4CtxLevel |= 1;
             }
+            cctxPtr->lz4CtxState = ctxTypeID;
         }
     }
 
-- 
cgit v0.12


From c852f20c39e877b877da49dea52dd4e36a5d6cb9 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 17:47:34 -0400
Subject: Switch ALLOC() to ALLOC_AND_ZERO() to Paper Over Existing
 Uninitialized Read

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 87e209f..b91cb7c 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -605,7 +605,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         if (cctxPtr->maxBufferSize < requiredBuffSize) {
             cctxPtr->maxBufferSize = 0;
             FREEMEM(cctxPtr->tmpBuff);
-            cctxPtr->tmpBuff = (BYTE*)ALLOC(requiredBuffSize);
+            cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize);
             if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             cctxPtr->maxBufferSize = requiredBuffSize;
     }   }
-- 
cgit v0.12


From 66b6fbfe6fa9d7e8d4686e6525883ff4aef31e02 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 14 Mar 2018 15:51:59 -0400
Subject: Restore the Other Old Streaming Functions in a Degraded Fashion

---
 lib/lz4.c | 19 +++++++++++++++++++
 lib/lz4.h | 11 +++++++++--
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 48cdad1..37782f5 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1707,6 +1707,25 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize,
 
 int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
 
+int LZ4_resetStreamState(void* state, char* inputBuffer)
+{
+    (void)inputBuffer;
+    LZ4_resetStream((LZ4_stream_t*)state);
+    return 0;
+}
+
+void* LZ4_create (char* inputBuffer)
+{
+    (void)inputBuffer;
+    return LZ4_createStream();
+}
+
+char* LZ4_slideInputBuffer (void* state)
+{
+    // avoid const char * -> char * conversion warning
+    return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary;
+}
+
 /* Obsolete streaming decompression functions */
 
 int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
diff --git a/lib/lz4.h b/lib/lz4.h
index e4a257b..6cb6589 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -477,8 +477,15 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
 LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
-/* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
+/* Broken, obsolete streaming functions; do not use!
+ *
+ * These functions depended on data that is no longer tracked in the state. They
+ * are therefore broken--they don't retain any history across compressions.
+ */
+LZ4_DEPRECATED("Broken!!! Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
+LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
+LZ4_DEPRECATED("Broken!!! Use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4_DEPRECATED("Broken!!! Use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
 
 /* Obsolete streaming decoding functions */
 LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
-- 
cgit v0.12


From b0a18896fe53cf2b69142c092f955e57b2b0d8df Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 14 Mar 2018 15:58:38 -0400
Subject: Move LZ4_compress_fast_extState_noReset Declaration to Unstable
 Section

---
 lib/lz4.h | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 6cb6589..d0ec204 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -179,21 +179,13 @@ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int d
 
 
 /*!
-LZ4_compress_fast_extState_noReset() :
 LZ4_compress_fast_extState() :
     Same compression function, just using an externally allocated memory space to store compression state.
     Use LZ4_sizeofState() to know how much memory must be allocated,
     and allocate it on 8-bytes boundaries (using malloc() typically).
     Then, provide it as 'void* state' to compression function.
-
-    Use the _noReset variant if LZ4_resetStream() was called on the state
-    buffer before being used for the first time (calls to both extState
-    functions leave the state in a safe state, so zeroing is not required
-    between calls). Otherwise, using the legacy _extState requires LZ4 to
-    reinitialize the state internally for every call.
 */
 LZ4LIB_API int LZ4_sizeofState(void);
-LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
@@ -351,6 +343,29 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 /*^**********************************************
  * !!!!!!   STATIC LINKING ONLY   !!!!!!
  ***********************************************/
+
+/*-************************************
+ *  Unstable declarations
+ **************************************
+ * Declarations in this section should be considered unstable.
+ * Use at your own peril, etc., etc.
+ * They may be removed in the future.
+ * Their signatures may change.
+ **************************************/
+
+/*!
+LZ4_compress_fast_extState_noReset() :
+    A variant of LZ4_compress_fast_extState().
+
+    Use the _noReset variant if LZ4_resetStream() was called on the state
+    buffer before being used for the first time (calls to both extState
+    functions leave the state in a safe state, so zeroing is not required
+    between calls). Otherwise, using the plain _extState requires LZ4 to
+    reinitialize the state internally for every call.
+*/
+LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+
+
 /*-************************************
  *  Private definitions
  **************************************
-- 
cgit v0.12


From 70f14823a46719e81e808d9ed9df90f478bcfd3f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 31 Jan 2018 18:11:37 -0500
Subject: Remove Framebench Tool

---
 tests/Makefile     |   6 +-
 tests/framebench.c | 295 -----------------------------------------------------
 2 files changed, 1 insertion(+), 300 deletions(-)
 delete mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index a351d35..5954370 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -62,7 +62,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen framebench
+all: fullbench fuzzer frametest datagen
 
 all32: CFLAGS+=-m32
 all32: all
@@ -98,9 +98,6 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
-	$(CC) $(FLAGS) $^ -o $@$(EXT)
-
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -112,7 +109,6 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
-        framebench$(EXT) \
         fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
diff --git a/tests/framebench.c b/tests/framebench.c
deleted file mode 100644
index fb4f38c..0000000
--- a/tests/framebench.c
+++ /dev/null
@@ -1,295 +0,0 @@
-#include <assert.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <time.h>
-#include <unistd.h>
-
-#include "lz4.h"
-#include "lz4frame.h"
-#include "lz4frame_static.h"
-
-#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
-
-typedef struct {
-  size_t iter;
-  LZ4_stream_t *ctx;
-  LZ4F_cctx *cctx;
-  char *obuf;
-  size_t osize;
-  const char* ibuf;
-  size_t isize;
-  size_t num_ibuf;
-  const LZ4F_CDict* cdict;
-  LZ4F_preferences_t* prefs;
-  const LZ4F_compressOptions_t* options;
-} bench_params_t;
-
-size_t compress_frame(bench_params_t *p) {
-  size_t iter = p->iter;
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressFrame_usingCDict(
-    cctx,
-    obuf,
-    osize,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
-    isize,
-    cdict,
-    prefs);
-  LZ4F_CHECK(oused);
-
-  return oused;
-}
-
-size_t compress_begin(bench_params_t *p) {
-  size_t iter = p->iter;
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-  const LZ4F_compressOptions_t* options = p->options;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressUpdate(
-    cctx,
-    obuf,
-    oend - obuf,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
-    isize,
-    options);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
-  LZ4F_CHECK(oused);
-
-  return obuf - p->obuf;
-}
-
-size_t compress_default(bench_params_t *p) {
-  size_t iter = p->iter;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_extState(bench_params_t *p) {
-  size_t iter = p->iter;
-  LZ4_stream_t *ctx = p->ctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_fast_extState_noReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-uint64_t bench(
-    size_t (*fun)(bench_params_t *),
-    uint64_t repetitions,
-    bench_params_t *params,
-    size_t *osizePtr
-) {
-  struct timespec start, end;
-  size_t i, osize = 0;
-
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
-
-  for (i = 0; i < repetitions; i++) {
-    size_t o;
-    params->iter = i;
-    o = fun(params);
-    if (!o) return 0;
-    osize += o;
-  }
-
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
-
-  *osizePtr = osize / repetitions;
-  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
-}
-
-int main(int argc, char *argv[]) {
-
-
-  struct stat st;
-  size_t bytes_read;
-
-  const char *dict_fn;
-  size_t dict_size;
-  char *dict_buf;
-  FILE *dict_file;
-
-  const char *in_fn;
-  size_t in_size;
-  size_t num_in_buf;
-  size_t cur_in_buf;
-  char *in_buf;
-  FILE *in_file;
-
-  size_t out_size;
-  char *out_buf;
-
-  LZ4_stream_t *ctx;
-  LZ4F_cctx *cctx;
-  LZ4F_CDict *cdict;
-  LZ4F_preferences_t prefs;
-  LZ4F_compressOptions_t options;
-
-  size_t out_used;
-
-  uint64_t time_taken;
-  uint64_t repetitions;
-
-  bench_params_t params;
-
-  if (argc != 3) return 1;
-  dict_fn = argv[1];
-  in_fn = argv[2];
-
-  if (stat(dict_fn, &st)) return 1;
-  dict_size = st.st_size;
-  dict_buf = (char *)malloc(dict_size);
-  if (!dict_buf) return 1;
-  dict_file = fopen(dict_fn, "r");
-  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
-  if (bytes_read != dict_size) return 1;
-
-  if (stat(in_fn, &st)) return 1;
-  in_size = st.st_size;
-  num_in_buf = 256 * 1024 * 1024 / in_size;
-  if (num_in_buf == 0) {
-    num_in_buf = 1;
-  }
-
-  in_buf = (char *)malloc(in_size * num_in_buf);
-  if (!in_buf) return 1;
-  in_file = fopen(in_fn, "r");
-  bytes_read = fread(in_buf, 1, in_size, in_file);
-  if (bytes_read != in_size) return 1;
-
-  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
-    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
-  }
-
-  if (in_size <= 1024) {
-    repetitions = 100000;
-  } else
-  if (in_size <= 16384) {
-    repetitions = 10000;
-  } else
-  if (in_size <= 131072) {
-    repetitions = 1000;
-  } else
-  if (in_size <= 1048576) {
-    repetitions = 100;
-  } else {
-    repetitions = 50;
-  }
-
-  memset(&prefs, 0, sizeof(prefs));
-  prefs.autoFlush = 1;
-  if (in_size < 64 * 1024)
-    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
-  prefs.frameInfo.contentSize = in_size;
-
-  memset(&options, 0, sizeof(options));
-  options.stableSrc = 1;
-
-  out_size = LZ4F_compressFrameBound(in_size, &prefs);
-  out_buf = (char *)malloc(out_size);
-  if (!out_buf) return 1;
-
-  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
-  if (cctx == NULL) return 1;
-
-  ctx = LZ4_createStream();
-  if (ctx == NULL) return 1;
-
-  cdict = LZ4F_createCDict(dict_buf, dict_size);
-  if (!cdict) return 1;
-
-  fprintf(stderr, "dict  size: %zd\n", dict_size);
-  fprintf(stderr, "input size: %zd\n", in_size);
-
-  params.ctx = ctx;
-  params.cctx = cctx;
-  params.obuf = out_buf;
-  params.osize = out_size;
-  params.ibuf = in_buf;
-  params.isize = in_size;
-  params.num_ibuf = num_in_buf;
-  params.cdict = NULL;
-  params.prefs = &prefs;
-  params.options = &options;
-
-  time_taken = bench(compress_default, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_extState, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  params.cdict = cdict;
-
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  return 0;
-}
\ No newline at end of file
-- 
cgit v0.12