From 0b876db6d42ec22da0c635e97a0d690908f2104a Mon Sep 17 00:00:00 2001
From: Yann Collet <yann.collet.73@gmail.com>
Date: Thu, 18 Apr 2019 16:06:02 -0700
Subject: address a few minor Visual warnings

and created target cxx17build
---
 Makefile            |  8 ++++++++
 doc/lz4_manual.html | 29 +++++++++++++----------------
 lib/lz4.h           |  8 +++++---
 lib/lz4hc.c         |  1 +
 tests/frametest.c   |  7 ++++---
 tests/fullbench.c   |  4 ++--
 tests/fuzzer.c      |  4 ++--
 7 files changed, 35 insertions(+), 26 deletions(-)

diff --git a/Makefile b/Makefile
index f3c6ce2..f3844a1 100644
--- a/Makefile
+++ b/Makefile
@@ -181,6 +181,14 @@ gpptest gpptest32: clean
 	CC=$(CC) $(MAKE) -C $(PRGDIR)  all CFLAGS="$(CFLAGS)"
 	CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
 
+cxx17build : CC = "$(CXX) -Wno-deprecated"
+cxx17build : CFLAGS = -std=c++17 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
+cxx17build : clean
+	$(CXX) -v
+	CC=$(CC) $(MAKE) -C $(LZ4DIR)  all CFLAGS="$(CFLAGS)"
+	CC=$(CC) $(MAKE) -C $(PRGDIR)  all CFLAGS="$(CFLAGS)"
+	CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
+
 ctocpptest: LIBCC="$(CC)"
 ctocpptest: TESTCC="$(CXX)"
 ctocpptest: CFLAGS=""
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 356a60d..ee43b8a 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -454,21 +454,18 @@ union LZ4_streamDecode_u {
  
 </p></pre><BR>
 
-<pre><b>LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") LZ4LIB_API
-int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
-LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") LZ4LIB_API
-int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
-LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") LZ4LIB_API
-int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
-</b><p>  These functions used to be a bit faster than LZ4_decompress_safe(),
-  but situation has changed in recent versions.
-  Now, `LZ4_decompress_safe()` is as fast and sometimes even faster than `LZ4_decompress_fast()`.
-  Moreover, LZ4_decompress_safe() is protected vs malformed input, while `LZ4_decompress_fast()` is not, making it a security liability.
+<pre><b></b><p>  These functions used to be faster than LZ4_decompress_safe(),
+  but it has changed, and they are now slower than LZ4_decompress_safe().
+  This is because LZ4_decompress_fast() doesn't know the input size,
+  and therefore must progress more cautiously in the input buffer to not read beyond the end of block.
+  On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.
   As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
 
-  Last LZ4_decompress_fast() specificity is that it can decompress a block without knowing its compressed size.
-  Note that even that functionality could be achieved in a more secure manner if need be,
-  though it would require new prototypes, and adaptation of the implementation to this new use case.
+  The last remaining LZ4_decompress_fast() specificity is that
+  it can decompress a block without knowing its compressed size.
+  Such functionality could be achieved in a more secure manner,
+  by also providing the maximum size of input buffer,
+  but it would require new prototypes, and adaptation of the implementation to this new use case.
 
   Parameters:
   originalSize : is the uncompressed size to regenerate.
@@ -477,9 +474,9 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
            The function expects to finish at block's end exactly.
            If the source stream is detected malformed, the function stops decoding and returns a negative result.
   note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer.
-         However, since it doesn't know its 'src' size, it may read an unknown amount of input, and overflow input buffer.
-         Also, since match offsets are not validated, match reads from 'src' may underflow.
-         These issues never happen if input data is correct.
+         However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds.
+         Also, since match offsets are not validated, match reads from 'src' may underflow too.
+         These issues never happen if input (compressed) data is correct.
          But they may happen if input data is invalid (error or intentional tampering).
          As a consequence, use these functions in trusted environments with trusted data **only**.
  
diff --git a/lib/lz4.h b/lib/lz4.h
index 962f5e6..737a0c7 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -638,9 +638,11 @@ LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4
  *  On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.
  *  As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
  *
- *  Only LZ4_decompress_fast() specificity is that it can decompress a block without knowing its compressed size.
- *  Even that functionality could be achieved in a more secure manner if need be,
- *  though it would require new prototypes, and adaptation of the implementation to this new use case.
+ *  The last remaining LZ4_decompress_fast() specificity is that
+ *  it can decompress a block without knowing its compressed size.
+ *  Such functionality could be achieved in a more secure manner,
+ *  by also providing the maximum size of input buffer,
+ *  but it would require new prototypes, and adaptation of the implementation to this new use case.
  *
  *  Parameters:
  *  originalSize : is the uncompressed size to regenerate.
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d5f6743..031df8f 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1396,6 +1396,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
              }   }
          }  /* for (cur = 1; cur <= last_match_pos; cur++) */
 
+         assert(last_match_pos < LZ4_OPT_NUM + TRAILING_LITERALS);
          best_mlen = opt[last_match_pos].mlen;
          best_off = opt[last_match_pos].off;
          cur = last_match_pos - best_mlen;
diff --git a/tests/frametest.c b/tests/frametest.c
index 9f7cb8d..bf95beb 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -667,8 +667,8 @@ int basicTests(U32 seed, double compressibility)
       for (blockSizeID = 4; blockSizeID < 8; ++blockSizeID) {
         result = LZ4F_getBlockSize(blockSizeID);
         CHECK(result);
-        DISPLAYLEVEL(3, "Returned block size of %zu bytes for blockID %u \n",
-                         result, blockSizeID);
+        DISPLAYLEVEL(3, "Returned block size of %u bytes for blockID %u \n",
+                         (unsigned)result, blockSizeID);
       }
 
       /* Test an invalid input that's too large */
@@ -770,7 +770,8 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un
     const BYTE* b2=(const BYTE*)buff2;
     DISPLAY("locateBuffDiff: looking for error position \n");
     if (nonContiguous) {
-        DISPLAY("mode %u: non-contiguous output (%zu bytes), cannot search \n", nonContiguous, size);
+        DISPLAY("mode %u: non-contiguous output (%u bytes), cannot search \n",
+                nonContiguous, (unsigned)size);
         return;
     }
     while (p < size && b1[p]==b2[p]) p++;
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 1a52aab..d2af662 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -162,13 +162,13 @@ static LZ4_stream_t LZ4_stream;
 static void local_LZ4_resetDictT(void)
 {
     void* const r = LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
-    assert(r != NULL);
+    assert(r != NULL); (void)r;
 }
 
 static void local_LZ4_createStream(void)
 {
     void* const r = LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
-    assert(r != NULL);
+    assert(r != NULL); (void)r;
 }
 
 static int local_LZ4_saveDict(const char* in, char* out, int inSize)
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 78f90a1..9908a7b 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -490,6 +490,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /* Test decompress_fast() with input buffer size exactly correct => must not read out of bound */
         {   char* const cBuffer_exact = (char*)malloc((size_t)compressedSize);
             assert(cBuffer_exact != NULL);
+            assert(compressedSize <= compressedBufferSize);
             memcpy(cBuffer_exact, compressedBuffer, compressedSize);
 
             /* Test decoding with output size exactly correct => must work */
@@ -1302,9 +1303,8 @@ static void FUZ_unitTests(int compressionLevel)
             int iNext = 0;
             int dNext = 0;
             int compressedSize;
-            size_t const testVerifySize = testInputSize;
 
-            assert((size_t)dBufferSize * 2 + 1 < testVerifySize);   /* space used by ringBufferSafe and ringBufferFast */
+            assert((size_t)dBufferSize * 2 + 1 < testInputSize);   /* space used by ringBufferSafe and ringBufferFast */
             XXH64_reset(&xxhOrig, 0);
             XXH64_reset(&xxhNewSafe, 0);
             XXH64_reset(&xxhNewFast, 0);
-- 
cgit v0.12


From 6cb084ed368ab8a4df3b48eb9683101c5e518418 Mon Sep 17 00:00:00 2001
From: Yann Collet <yann.collet.73@gmail.com>
Date: Thu, 18 Apr 2019 16:41:27 -0700
Subject: fuzzer: reduce stack usage to please Visual static analyzer

---
 tests/fuzzer.c | 124 ++++++++++++++++++++++++++++++---------------------------
 1 file changed, 66 insertions(+), 58 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 9908a7b..a99f2f3 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -319,8 +319,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
     void* const lowAddrBuffer = FUZ_createLowAddr(labSize);
     void* const stateLZ4   = malloc((size_t)LZ4_sizeofState());
     void* const stateLZ4HC = malloc((size_t)LZ4_sizeofStateHC());
-    LZ4_stream_t LZ4dict;
-    LZ4_streamHC_t LZ4dictHC;
+    LZ4_stream_t LZ4dictBody;
+    LZ4_streamHC_t* LZ4dictHC = LZ4_createStreamHC();
     U32 coreRandState = seed;
     clock_t const clockStart = clock();
     clock_t const clockDuration = (clock_t)duration_s * CLOCKS_PER_SEC;
@@ -346,12 +346,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
 
     /* init */
-    if(!CNBuffer || !compressedBuffer || !decodedBuffer) {
+    if(!CNBuffer || !compressedBuffer || !decodedBuffer || !LZ4dictHC) {
         DISPLAY("Not enough memory to start fuzzer tests");
         exit(1);
     }
-    if ( LZ4_initStream(&LZ4dict, sizeof(LZ4dict)) == NULL) abort();
-    if ( LZ4_initStreamHC(&LZ4dictHC, sizeof(LZ4dictHC)) == NULL) abort();
+    if ( LZ4_initStream(&LZ4dictBody, sizeof(LZ4dictBody)) == NULL) abort();
     {   U32 randState = coreRandState ^ PRIME3;
         FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
     }
@@ -455,42 +454,42 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
         /* Test compression HC */
         FUZ_DISPLAYTEST("test LZ4_compress_HC()");
-        ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed");
-        HCcompressedSize = ret;
+        HCcompressedSize = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
+        FUZ_CHECKTEST(HCcompressedSize==0, "LZ4_compress_HC() failed");
 
         /* Test compression HC using external state */
         FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()");
-        ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed")
+        {   int const r = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
+            FUZ_CHECKTEST(r==0, "LZ4_compress_HC_extStateHC() failed")
+        }
 
         /* Test compression HC using fast reset external state */
         FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC_fastReset()");
-        ret = LZ4_compress_HC_extStateHC_fastReset(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC_fastReset() failed");
+        {   int const r = LZ4_compress_HC_extStateHC_fastReset(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
+            FUZ_CHECKTEST(r==0, "LZ4_compress_HC_extStateHC_fastReset() failed");
+        }
 
         /* Test compression using external state */
         FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()");
-        ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
+        {   int const r = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
+            FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState() failed"); }
 
         /* Test compression using fast reset external state*/
         FUZ_DISPLAYTEST();
-        ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed");
+        {   int const r = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
+            FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState_fastReset() failed"); }
 
         /* Test compression */
         FUZ_DISPLAYTEST("test LZ4_compress_default()");
-        ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
-        FUZ_CHECKTEST(ret<=0, "LZ4_compress_default() failed");
-        compressedSize = ret;
+        compressedSize = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
+        FUZ_CHECKTEST(compressedSize<=0, "LZ4_compress_default() failed");
 
         /* Decompression tests */
 
         /* Test decompress_fast() with input buffer size exactly correct => must not read out of bound */
         {   char* const cBuffer_exact = (char*)malloc((size_t)compressedSize);
             assert(cBuffer_exact != NULL);
-            assert(compressedSize <= compressedBufferSize);
+            assert(compressedSize <= (int)compressedBufferSize);
             memcpy(cBuffer_exact, compressedBuffer, compressedSize);
 
             /* Test decoding with output size exactly correct => must work */
@@ -685,20 +684,20 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary");
         dict -= (size_t)(FUZ_rand(&randState) & 0xF) + 1;   /* create space, so now dictionary is an ExtDict */
         if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
-        LZ4_loadDict(&LZ4dict, dict, dictSize);
-        blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+        LZ4_loadDict(&LZ4dictBody, dict, dictSize);
+        blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
         FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
 
         FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary but with an output buffer too short by one byte");
-        LZ4_loadDict(&LZ4dict, dict, dictSize);
-        ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
+        LZ4_loadDict(&LZ4dictBody, dict, dictSize);
+        ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
         FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
 
         FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()");
         DISPLAYLEVEL(5, " compress %i bytes from buffer(%p) into dst(%p) using dict(%p) of size %i \n",
                      blockSize, (const void *)block, (void *)decodedBuffer, (const void *)dict, dictSize);
-        LZ4_loadDict(&LZ4dict, dict, dictSize);
-        ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
+        LZ4_loadDict(&LZ4dictBody, dict, dictSize);
+        ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
         FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
 
@@ -754,15 +753,15 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             U32 expectedCrc;
 
             FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_loadDict()");
-            LZ4_loadDict(&LZ4dict, dict, dictSize);
-            expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+            LZ4_loadDict(&LZ4dictBody, dict, dictSize);
+            expectedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
             FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
             expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
 
             FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()");
-            LZ4_loadDict(&LZ4dict, dict, dictSize);
+            LZ4_loadDict(&LZ4dictBody, dict, dictSize);
             LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
-            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
             blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
             FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good");
@@ -777,14 +776,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
             FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short");
             LZ4_resetStream_fast(&LZ4_stream);
-            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
             FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
             /* note : context is no longer dirty after a failed compressed block */
 
             FUZ_DISPLAYTEST();
             LZ4_resetStream_fast(&LZ4_stream);
-            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
@@ -794,7 +793,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
             FUZ_DISPLAYTEST();
             LZ4_resetStream_fast(&LZ4_stream);
-            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
@@ -850,24 +849,24 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_DISPLAYTEST("LZ4_compress_HC_continue with an external dictionary");
         dict -= (FUZ_rand(&randState) & 7);    /* even bigger separation */
         if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
-        LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
-        LZ4_setCompressionLevel (&LZ4dictHC, compressionLevel);
-        blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
+        LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
+        LZ4_setCompressionLevel (LZ4dictHC, compressionLevel);
+        blockContinueCompressedSize = LZ4_compress_HC_continue(LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
         FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed");
-        FUZ_CHECKTEST(LZ4dictHC.internal_donotuse.dirty, "Context should be clean");
+        FUZ_CHECKTEST(LZ4dictHC->internal_donotuse.dirty, "Context should be clean");
 
         FUZ_DISPLAYTEST("LZ4_compress_HC_continue with same external dictionary, but output buffer 1 byte too short");
-        LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
-        ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+        LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
+        ret = LZ4_compress_HC_continue(LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
         FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (expected %i, but result=%i)", blockContinueCompressedSize, ret);
         /* note : context is no longer dirty after a failed compressed block */
 
         FUZ_DISPLAYTEST("LZ4_compress_HC_continue with same external dictionary, and output buffer exactly the right size");
-        LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
-        ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+        LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
+        ret = LZ4_compress_HC_continue(LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different : ret(%i) != expected(%i)", ret, blockContinueCompressedSize);
         FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer");
-        FUZ_CHECKTEST(LZ4dictHC.internal_donotuse.dirty, "Context should be clean");
+        FUZ_CHECKTEST(LZ4dictHC->internal_donotuse.dirty, "Context should be clean");
 
         FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
@@ -886,8 +885,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             LZ4_streamHC_t LZ4_streamHC;
             LZ4_initStreamHC(&LZ4_streamHC, sizeof(LZ4_streamHC));
 
-            LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
-            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, LZ4dictHC);
             LZ4_setCompressionLevel (&LZ4_streamHC, compressionLevel);
             blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed");
@@ -895,14 +894,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
             FUZ_DISPLAYTEST();
             LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
-            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, LZ4dictHC);
             ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
             FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
             /* note : context is no longer dirty after a failed compressed block */
 
             FUZ_DISPLAYTEST();
             LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
-            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, LZ4dictHC);
             ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer");
@@ -910,7 +909,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
             FUZ_DISPLAYTEST();
             LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
-            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, LZ4dictHC);
             ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer");
@@ -933,9 +932,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         {   int const availableSpace = (int)(FUZ_rand(&randState) % blockSize) + 5;
             int consumedSize = blockSize;
             FUZ_DISPLAYTEST();
-            LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
-            LZ4_setCompressionLevel(&LZ4dictHC, compressionLevel);
-            blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
+            LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
+            LZ4_setCompressionLevel(LZ4dictHC, compressionLevel);
+            blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
             DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed");
             FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow");
@@ -975,6 +974,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
     free(compressedBuffer);
     free(decodedBuffer);
     FUZ_freeLowAddr(lowAddrBuffer, labSize);
+    LZ4_freeStreamHC(LZ4dictHC);
     free(stateLZ4);
     free(stateLZ4HC);
     return result;
@@ -990,28 +990,31 @@ static void FUZ_unitTests(int compressionLevel)
     const unsigned testNb = 0;
     const unsigned seed   = 0;
     const unsigned cycleNb= 0;
-    char testInput[testInputSize];
-    char testCompressed[testCompressedSize];
-    char testVerify[testInputSize];
+    char* testInput = (char*)malloc(testInputSize);
+    char* testCompressed = (char*)malloc(testCompressedSize);
+    char* testVerify = (char*)malloc(testInputSize);
     char ringBuffer[ringBufferSize] = {0};
     U32 randState = 1;
 
     /* Init */
+    if (!testInput || !testCompressed || !testVerify) {
+        EXIT_MSG("not enough memory for FUZ_unitTests");
+    }
     FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
 
     /* 32-bits address space overflow test */
     FUZ_AddressOverflow();
 
     /* LZ4 streaming tests */
-    {   LZ4_stream_t* statePtr;
-        LZ4_stream_t  streamingState;
+    {   LZ4_stream_t  streamingState;
         U64 crcOrig;
         int result;
 
         /* Allocation test */
-        statePtr = LZ4_createStream();
-        FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
-        LZ4_freeStream(statePtr);
+        {   LZ4_stream_t* const statePtr = LZ4_createStream();
+            FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
+            LZ4_freeStream(statePtr);
+        }
 
         /* simple compression test */
         crcOrig = XXH64(testInput, testCompressedSize, 0);
@@ -1389,6 +1392,11 @@ static void FUZ_unitTests(int compressionLevel)
         }
     }
 
+    /* clean up */
+    free(testInput);
+    free(testCompressed);
+    free(testVerify);
+
     printf("All unit tests completed successfully compressionLevel=%d \n", compressionLevel);
     return;
 }
-- 
cgit v0.12


From 4790994568d4d3ba492857c4c567ea6517588a45 Mon Sep 17 00:00:00 2001
From: Yann Collet <yann.collet.73@gmail.com>
Date: Thu, 18 Apr 2019 17:12:14 -0700
Subject: fuzzer : reduced stack usage

---
 tests/fuzzer.c | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a99f2f3..ffbeefc 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -881,39 +881,39 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
         /* Compress HC using external dictionary stream */
         FUZ_DISPLAYTEST();
-        {
-            LZ4_streamHC_t LZ4_streamHC;
-            LZ4_initStreamHC(&LZ4_streamHC, sizeof(LZ4_streamHC));
+        {   LZ4_streamHC_t* LZ4_streamHC = LZ4_createStreamHC();
 
             LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
-            LZ4_attach_HC_dictionary(&LZ4_streamHC, LZ4dictHC);
-            LZ4_setCompressionLevel (&LZ4_streamHC, compressionLevel);
-            blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
+            LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
+            LZ4_setCompressionLevel (LZ4_streamHC, compressionLevel);
+            blockContinueCompressedSize = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed");
-            FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean");
+            FUZ_CHECKTEST(LZ4_streamHC->internal_donotuse.dirty, "Context should be clean");
 
             FUZ_DISPLAYTEST();
-            LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
-            LZ4_attach_HC_dictionary(&LZ4_streamHC, LZ4dictHC);
-            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+            LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
+            ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
             FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
             /* note : context is no longer dirty after a failed compressed block */
 
             FUZ_DISPLAYTEST();
-            LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
-            LZ4_attach_HC_dictionary(&LZ4_streamHC, LZ4dictHC);
-            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+            LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
+            ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer");
-            FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean");
+            FUZ_CHECKTEST(LZ4_streamHC->internal_donotuse.dirty, "Context should be clean");
 
             FUZ_DISPLAYTEST();
-            LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
-            LZ4_attach_HC_dictionary(&LZ4_streamHC, LZ4dictHC);
-            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+            LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
+            ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer");
-            FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean");
+            FUZ_CHECKTEST(LZ4_streamHC->internal_donotuse.dirty, "Context should be clean");
+
+            LZ4_freeStreamHC(LZ4_streamHC);
         }
 
         FUZ_DISPLAYTEST();
-- 
cgit v0.12


From 4e4f1ad623cca9ebd212400b5783c63fd76dc868 Mon Sep 17 00:00:00 2001
From: Yann Collet <yann.collet.73@gmail.com>
Date: Thu, 18 Apr 2019 17:26:01 -0700
Subject: ensure list of names is large enough

---
 programs/util.h | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/programs/util.h b/programs/util.h
index 6a35481..1dd515c 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -379,7 +379,7 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi
 */
 UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size)
 {
-    void* newptr = realloc(ptr, size);
+    void* const newptr = realloc(ptr, size);
     if (newptr) return newptr;
     free(ptr);
     return NULL;
@@ -529,7 +529,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_
  * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
  */
 UTIL_STATIC const char**
-UTIL_createFileList(const char** inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb)
+UTIL_createFileList(const char** inputNames, unsigned inputNamesNb,
+                    char** allocatedBuffer, unsigned* allocatedNamesNb)
 {
     size_t pos;
     unsigned i, nbFiles;
@@ -543,16 +544,14 @@ UTIL_createFileList(const char** inputNames, unsigned inputNamesNb, char** alloc
         if (!UTIL_isDirectory(inputNames[i])) {
             size_t const len = strlen(inputNames[i]);
             if (pos + len >= bufSize) {
-                size_t newListSize = bufSize + LIST_SIZE_INCREASE;
-                buf = (char*)UTIL_realloc(buf, newListSize);
-                bufSize = newListSize;
+                while (pos + len >= bufSize) bufSize += LIST_SIZE_INCREASE;
+                buf = (char*)UTIL_realloc(buf, bufSize);
                 if (!buf) return NULL;
             }
-            if (pos + len < bufSize) {
-                strncpy(buf + pos, inputNames[i], bufSize - pos);
-                pos += len + 1;
-                nbFiles++;
-            }
+            assert(pos + len < bufSize);
+            strncpy(buf + pos, inputNames[i], bufSize - pos);
+            pos += len + 1;
+            nbFiles++;
         } else {
             char* bufend = buf + bufSize;
             nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend);
-- 
cgit v0.12