fixed an old bug in LZ4F_flush()

which remained undetected so far,
as it requires a fairly large number of conditions to be triggered,
starting with enabling Block checksum, which is disabled by default,
and which usage is known to be extremely rare.

2 files changed, 109 insertions, 66 deletions

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 3f81ef0..2164470 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -74,7 +74,6 @@ You can contact the author at :
 #endif
 
 
-
 /*-************************************
 *  Includes
 **************************************/
@@ -139,8 +138,8 @@ static U32 LZ4F_readLE32 (const void* src)
 {
     const BYTE* const srcPtr = (const BYTE*)src;
     U32 value32 = srcPtr[0];
-    value32 += (srcPtr[1]<<8);
-    value32 += (srcPtr[2]<<16);
+    value32 += ((U32)srcPtr[1])<< 8;
+    value32 += ((U32)srcPtr[2])<<16;
     value32 += ((U32)srcPtr[3])<<24;
     return value32;
 }
@@ -204,7 +203,8 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
 
 static const size_t minFHSize = 7;
 static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX;   /* 19 */
-static const size_t BHSize = 4;
+static const size_t BHSize = 4;  /* block header : size, and compress flag */
+static const size_t BFSize = 4;  /* block footer : checksum (optional) */
 
 
 /*-************************************
@@ -329,11 +329,10 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
         size_t const lastBlockSize = flush ? partialBlockSize : 0;
         unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0);
 
-        size_t const blockHeaderSize = 4;
-        size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag;
-        size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
+        size_t const blockCRCSize = BFSize * prefsPtr->frameInfo.blockChecksumFlag;
+        size_t const frameEnd = BHSize + (prefsPtr->frameInfo.contentChecksumFlag*BFSize);
 
-        return ((blockHeaderSize + blockCRCSize) * nbBlocks) +
+        return ((BHSize + blockCRCSize) * nbBlocks) +
                (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;
     }
 }
@@ -394,15 +393,18 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
       if (LZ4F_isError(headerSize)) return headerSize;
       dstPtr += headerSize;   /* header size */ }
 
-    { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
+    assert(dstEnd >= dstPtr);
+    { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options);
       if (LZ4F_isError(cSize)) return cSize;
       dstPtr += cSize; }
 
-    { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, &options);   /* flush last block, and generate suffix */
+    assert(dstEnd >= dstPtr);
+    { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options);   /* flush last block, and generate suffix */
       if (LZ4F_isError(tailSize)) return tailSize;
       dstPtr += tailSize; }
 
-    return (dstPtr - dstStart);
+    assert(dstEnd >= dstStart);
+    return (size_t)(dstPtr - dstStart);
 }
 
 
@@ -662,7 +664,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     *dstPtr++ = (BYTE)(((1 & _2BITS) << 6)    /* Version('01') */
         + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
         + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
-        + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
+        + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
         + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
         +  (cctxPtr->prefs.frameInfo.dictID > 0) );
     /* BD Byte */
@@ -679,11 +681,11 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         dstPtr += 4;
     }
     /* Header CRC Byte */
-    *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
+    *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));
     dstPtr++;
 
     cctxPtr->cStage = 1;   /* header written, now request input data block */
-    return (dstPtr - dstStart);
+    return (size_t)(dstPtr - dstStart);
 }
 
 
@@ -718,27 +720,31 @@ typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize
 
 
 /*! LZ4F_makeBlock():
- *  compress a single block, add header and checksum
- *  assumption : dst buffer capacity is >= srcSize */
-static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
+ *  compress a single block, add header and optional checksum.
+ *  assumption : dst buffer capacity is >= BHSize + srcSize + crcSize
+ */
+static size_t LZ4F_makeBlock(void* dst,
+                       const void* src, size_t srcSize,
                              compressFunc_t compress, void* lz4ctx, int level,
-                             const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag)
+                       const LZ4F_CDict* cdict,
+                             LZ4F_blockChecksum_t crcFlag)
 {
     BYTE* const cSizePtr = (BYTE*)dst;
-    U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4),
+    U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
                                       (int)(srcSize), (int)(srcSize-1),
                                       level, cdict);
-    LZ4F_writeLE32(cSizePtr, cSize);
     if (cSize == 0) {  /* compression failed */
         cSize = (U32)srcSize;
         LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
-        memcpy(cSizePtr+4, src, srcSize);
+        memcpy(cSizePtr+BHSize, src, srcSize);
+    } else {
+        LZ4F_writeLE32(cSizePtr, cSize);
     }
     if (crcFlag) {
-        U32 const crc32 = XXH32(cSizePtr+4, cSize, 0);  /* checksum of compressed data */
-        LZ4F_writeLE32(cSizePtr+4+cSize, crc32);
+        U32 const crc32 = XXH32(cSizePtr+BHSize, cSize, 0);  /* checksum of compressed data */
+        LZ4F_writeLE32(cSizePtr+BHSize+cSize, crc32);
     }
-    return 4 + cSize + ((U32)crcFlag)*4;
+    return BHSize + cSize + ((U32)crcFlag)*BFSize;
 }
 
 
@@ -838,9 +844,11 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
             memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
             srcPtr += sizeToCopy;
 
-            dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize,
+            dstPtr += LZ4F_makeBlock(dstPtr,
+                                     cctxPtr->tmpIn, blockSize,
                                      compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
-                                     cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
+                                     cctxPtr->cdict,
+                                     cctxPtr->prefs.frameInfo.blockChecksumFlag);
 
             if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
             cctxPtr->tmpInSize = 0;
@@ -850,18 +858,22 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     while ((size_t)(srcEnd - srcPtr) >= blockSize) {
         /* compress full blocks */
         lastBlockCompressed = fromSrcBuffer;
-        dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize,
+        dstPtr += LZ4F_makeBlock(dstPtr,
+                                 srcPtr, blockSize,
                                  compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
-                                 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
+                                 cctxPtr->cdict,
+                                 cctxPtr->prefs.frameInfo.blockChecksumFlag);
         srcPtr += blockSize;
     }
 
     if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
         /* compress remaining input < blockSize */
         lastBlockCompressed = fromSrcBuffer;
-        dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr,
+        dstPtr += LZ4F_makeBlock(dstPtr,
+                                 srcPtr, (size_t)(srcEnd - srcPtr),
                                  compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
-                                 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
+                                 cctxPtr->cdict,
+                                 cctxPtr->prefs.frameInfo.blockChecksumFlag);
         srcPtr  = srcEnd;
     }
 
@@ -887,7 +899,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     /* some input data left, necessarily < blockSize */
     if (srcPtr < srcEnd) {
         /* fill tmp buffer */
-        size_t const sizeToCopy = srcEnd - srcPtr;
+        size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
         memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
         cctxPtr->tmpInSize = sizeToCopy;
     }
@@ -896,19 +908,21 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
         (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
 
     cctxPtr->totalInSize += srcSize;
-    return dstPtr - dstStart;
+    return (size_t)(dstPtr - dstStart);
 }
 
 
 /*! LZ4F_flush() :
- *  Should you need to create compressed data immediately, without waiting for a block to be filled,
- *  you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
- *  The result of the function is the number of bytes written into dstBuffer
- *  (it can be zero, this means there was no data left within compressionContext)
+ *  When compressed data must be sent immediately, without waiting for a block to be filled,
+ *  invoke LZ4_flush(), which will immediately compress any remaining data stored within LZ4F_cctx.
+ *  The result of the function is the number of bytes written into dstBuffer.
+ *  It can be zero, this means there was no data left within LZ4F_cctx.
  *  The function outputs an error code if it fails (can be tested using LZ4F_isError())
- *  The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ *  LZ4F_compressOptions_t* is optional. NULL is a valid argument.
  */
-size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr)
+size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
+                  void* dstBuffer, size_t dstCapacity,
+            const LZ4F_compressOptions_t* compressOptionsPtr)
 {
     BYTE* const dstStart = (BYTE*)dstBuffer;
     BYTE* dstPtr = dstStart;
@@ -916,26 +930,32 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
 
     if (cctxPtr->tmpInSize == 0) return 0;   /* nothing to flush */
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
-    if (dstCapacity < (cctxPtr->tmpInSize + 4)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);   /* +4 : block header(4)  */
+    if (dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize))
+        return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
     (void)compressOptionsPtr;   /* not yet useful */
 
     /* select compression function */
     compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
     /* compress tmp buffer */
-    dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize,
+    dstPtr += LZ4F_makeBlock(dstPtr,
+                             cctxPtr->tmpIn, cctxPtr->tmpInSize,
                              compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
-                             cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
-    if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
+                             cctxPtr->cdict,
+                             cctxPtr->prefs.frameInfo.blockChecksumFlag);
+    assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) < dstCapacity));
+
+    if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)
+        cctxPtr->tmpIn += cctxPtr->tmpInSize;
     cctxPtr->tmpInSize = 0;
 
     /* keep tmpIn within limits */
     if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) {  /* necessarily LZ4F_blockLinked */
-        int realDictSize = LZ4F_localSaveDict(cctxPtr);
+        int const realDictSize = LZ4F_localSaveDict(cctxPtr);
         cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
     }
 
-    return dstPtr - dstStart;
+    return (size_t)(dstPtr - dstStart);
 }
 
 
@@ -981,7 +1001,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
             return err0r(LZ4F_ERROR_frameSize_wrong);
     }
 
-    return dstPtr - dstStart;
+    return (size_t)(dstPtr - dstStart);
 }
 
 
@@ -1433,7 +1453,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
 
         /* decode block header */
             {   size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
-                size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4;
+                size_t const crcSize = dctx->frameInfo.blockChecksumFlag * BFSize;
                 if (nextCBlockSize==0) {  /* frameEnd signal, no more block */
                     dctx->dStage = dstage_getSuffix;
                     break;
@@ -1453,7 +1473,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 dctx->tmpInTarget = nextCBlockSize + crcSize;
                 dctx->dStage = dstage_getCBlock;
                 if (dstPtr==dstEnd) {
-                    nextSrcSizeHint = nextCBlockSize + crcSize + BHSize;
+                    nextSrcSizeHint = BHSize + nextCBlockSize + crcSize;
                     doAnotherStage = 0;
                 }
                 break;
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 3324f13..5e7cb11 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -110,23 +110,43 @@ LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void);
 LZ4LIB_API int             LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
 
 /*
-  These functions compress data in successive blocks of any size, using previous blocks as dictionary.
+  These functions compress data in successive blocks of any size,
+  using previous blocks as dictionary, to improve compression ratio.
   One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks.
   There is an exception for ring buffers, which can be smaller than 64 KB.
-  Ring buffers scenario is automatically detected and handled by LZ4_compress_HC_continue().
+  Ring-buffer scenario is automatically detected and handled within LZ4_compress_HC_continue().
 
-  Before starting compression, state must be properly initialized, using LZ4_resetStreamHC().
-  A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional).
+  Before starting compression, state must be allocated and properly initialized.
+  LZ4_createStreamHC() does both, though compression level is set to LZ4HC_CLEVEL_DEFAULT.
 
-  Then, use LZ4_compress_HC_continue() to compress each successive block.
-  Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
-  'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), to ensure operation success.
-  In case of failure, the API does not guarantee context recovery, therefore context must be reset, using `LZ4_resetStreamHC()`.
-  If `dst` buffer size cannot be made >= LZ4_compressBound(), consider using LZ4_compress_HC_continue_destSize() instead.
+  Selecting the compression level can be done with LZ4_resetStreamHC_fast() (starts a new stream)
+  or LZ4_setCompressionLevel() (anytime, between blocks in the same stream) (experimental).
+  LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once.
 
-  If, for any reason, previous data block can't be preserved unmodified in memory for next compression block,
-  you can save it to a more stable memory space, using LZ4_saveDictHC().
-  Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
+  If state space is provided manually with no guarantee of its content, for example allocated on stack,
+  it must be fully initialized, using LZ4_resetStreamHC().
+  LZ4_resetStreamHC() is heavier, and it's guaranteed to succeed on any valid memory segment.
+  In contrast, LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once.
+
+  After reset, a first "fictional block" can be designated as initial dictionary,
+  using LZ4_loadDictHC() (Optional).
+
+  Invoke LZ4_compress_HC_continue() to compress each successive block.
+  The number of blocks is unlimited.
+  Previous input blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
+
+  'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), ensuring compression success.
+  In case of failure, the API does not guarantee recovery, so the state _must_ be reset.
+  Whenever `dst` buffer size cannot be made >= LZ4_compressBound(),
+  consider using LZ4_compress_HC_continue_destSize() to ensure success.
+
+  Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks,
+  it's possible to copy the last blocks into a more stable memory space, using LZ4_saveDictHC().
+  Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer' (<= 64 KB)
+
+  After completing a streaming compression,
+  it's possible to start a new stream and re-use the LZ4_streamHC_t state 
+  by resetting it, using LZ4_resetStreamHC_fast().
 */
 
 LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel);
@@ -136,21 +156,22 @@ LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr,
                                    const char* src, char* dst,
                                          int srcSize, int maxDstSize);
 
-LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
-
 /*! LZ4_compress_HC_continue_destSize() : v1.9.0+
- *  Similar as LZ4_compress_HC_continue(),
+ *  Similar to LZ4_compress_HC_continue(),
  *  but will read as much data as possible from `src`
  *  to fit into `targetDstSize` budget.
  *  Result is provided into 2 parts :
  * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize)
  *           or 0 if compression fails.
- * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`.
+ * `srcSizePtr` : on success, *srcSizePtr will be updated to indicate how much bytes were read from `src`.
+ *           Note that this function may not consume the entire input.
  */
 LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
                                            const char* src, char* dst,
                                                  int* srcSizePtr, int targetDstSize);
 
+LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
+
 
 
 /*^**********************************************
@@ -293,14 +314,16 @@ extern "C" {
 #endif
 
 /*! LZ4_setCompressionLevel() : v1.8.0+ (experimental)
- *  It's possible to change compression level between 2 invocations of LZ4_compress_HC_continue*()
+ *  It's possible to change compression level
+ *  between successive invocations of LZ4_compress_HC_continue*()
+ *  for dynamic adaptation.
  */
 LZ4LIB_STATIC_API void LZ4_setCompressionLevel(
     LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
 /*! LZ4_favorDecompressionSpeed() : v1.8.2+ (experimental)
- *  Parser will select decisions favoring decompression over compression ratio.
- *  Only work at highest compression settings (level >= LZ4HC_CLEVEL_OPT_MIN)
+ *  Opt. Parser will favor decompression speed over compression ratio.
+ *  Only applicable to levels >= LZ4HC_CLEVEL_OPT_MIN.
  */
 LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed(
     LZ4_streamHC_t* LZ4_streamHCPtr, int favor);