From b5bedefabe513c810e77a64c448b00377e0a8bf0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 3 Apr 2019 16:28:42 -0700 Subject: fixed an old bug in LZ4F_flush() which remained undetected so far, as it requires a fairly large number of conditions to be triggered, starting with enabling Block checksum, which is disabled by default, and which usage is known to be extremely rare. --- lib/lz4frame.c | 114 +++++++++++++++++++++++++++++++++------------------------ lib/lz4hc.h | 61 ++++++++++++++++++++---------- 2 files changed, 109 insertions(+), 66 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 3f81ef0..2164470 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -74,7 +74,6 @@ You can contact the author at : #endif - /*-************************************ * Includes **************************************/ @@ -139,8 +138,8 @@ static U32 LZ4F_readLE32 (const void* src) { const BYTE* const srcPtr = (const BYTE*)src; U32 value32 = srcPtr[0]; - value32 += (srcPtr[1]<<8); - value32 += (srcPtr[2]<<16); + value32 += ((U32)srcPtr[1])<< 8; + value32 += ((U32)srcPtr[2])<<16; value32 += ((U32)srcPtr[3])<<24; return value32; } @@ -204,7 +203,8 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) static const size_t minFHSize = 7; static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ -static const size_t BHSize = 4; +static const size_t BHSize = 4; /* block header : size, and compress flag */ +static const size_t BFSize = 4; /* block footer : checksum (optional) */ /*-************************************ @@ -329,11 +329,10 @@ static size_t LZ4F_compressBound_internal(size_t srcSize, size_t const lastBlockSize = flush ? partialBlockSize : 0; unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); - size_t const blockHeaderSize = 4; - size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag; - size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + size_t const blockCRCSize = BFSize * prefsPtr->frameInfo.blockChecksumFlag; + size_t const frameEnd = BHSize + (prefsPtr->frameInfo.contentChecksumFlag*BFSize); - return ((blockHeaderSize + blockCRCSize) * nbBlocks) + + return ((BHSize + blockCRCSize) * nbBlocks) + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd; } } @@ -394,15 +393,18 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx, if (LZ4F_isError(headerSize)) return headerSize; dstPtr += headerSize; /* header size */ } - { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); + assert(dstEnd >= dstPtr); + { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options); if (LZ4F_isError(cSize)) return cSize; dstPtr += cSize; } - { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ + assert(dstEnd >= dstPtr); + { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */ if (LZ4F_isError(tailSize)) return tailSize; dstPtr += tailSize; } - return (dstPtr - dstStart); + assert(dstEnd >= dstStart); + return (size_t)(dstPtr - dstStart); } @@ -662,7 +664,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4) - + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3) + + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3) + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) + (cctxPtr->prefs.frameInfo.dictID > 0) ); /* BD Byte */ @@ -679,11 +681,11 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, dstPtr += 4; } /* Header CRC Byte */ - *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart); + *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart)); dstPtr++; cctxPtr->cStage = 1; /* header written, now request input data block */ - return (dstPtr - dstStart); + return (size_t)(dstPtr - dstStart); } @@ -718,27 +720,31 @@ typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize /*! LZ4F_makeBlock(): - * compress a single block, add header and checksum - * assumption : dst buffer capacity is >= srcSize */ -static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, + * compress a single block, add header and optional checksum. + * assumption : dst buffer capacity is >= BHSize + srcSize + crcSize + */ +static size_t LZ4F_makeBlock(void* dst, + const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level, - const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag) + const LZ4F_CDict* cdict, + LZ4F_blockChecksum_t crcFlag) { BYTE* const cSizePtr = (BYTE*)dst; - U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), + U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize), (int)(srcSize), (int)(srcSize-1), level, cdict); - LZ4F_writeLE32(cSizePtr, cSize); if (cSize == 0) { /* compression failed */ cSize = (U32)srcSize; LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); - memcpy(cSizePtr+4, src, srcSize); + memcpy(cSizePtr+BHSize, src, srcSize); + } else { + LZ4F_writeLE32(cSizePtr, cSize); } if (crcFlag) { - U32 const crc32 = XXH32(cSizePtr+4, cSize, 0); /* checksum of compressed data */ - LZ4F_writeLE32(cSizePtr+4+cSize, crc32); + U32 const crc32 = XXH32(cSizePtr+BHSize, cSize, 0); /* checksum of compressed data */ + LZ4F_writeLE32(cSizePtr+BHSize+cSize, crc32); } - return 4 + cSize + ((U32)crcFlag)*4; + return BHSize + cSize + ((U32)crcFlag)*BFSize; } @@ -838,9 +844,11 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); srcPtr += sizeToCopy; - dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize, + dstPtr += LZ4F_makeBlock(dstPtr, + cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; @@ -850,18 +858,22 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, while ((size_t)(srcEnd - srcPtr) >= blockSize) { /* compress full blocks */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize, + dstPtr += LZ4F_makeBlock(dstPtr, + srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); srcPtr += blockSize; } if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { /* compress remaining input < blockSize */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr, + dstPtr += LZ4F_makeBlock(dstPtr, + srcPtr, (size_t)(srcEnd - srcPtr), compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); srcPtr = srcEnd; } @@ -887,7 +899,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, /* some input data left, necessarily < blockSize */ if (srcPtr < srcEnd) { /* fill tmp buffer */ - size_t const sizeToCopy = srcEnd - srcPtr; + size_t const sizeToCopy = (size_t)(srcEnd - srcPtr); memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); cctxPtr->tmpInSize = sizeToCopy; } @@ -896,19 +908,21 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); cctxPtr->totalInSize += srcSize; - return dstPtr - dstStart; + return (size_t)(dstPtr - dstStart); } /*! LZ4F_flush() : - * Should you need to create compressed data immediately, without waiting for a block to be filled, - * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. - * The result of the function is the number of bytes written into dstBuffer - * (it can be zero, this means there was no data left within compressionContext) + * When compressed data must be sent immediately, without waiting for a block to be filled, + * invoke LZ4_flush(), which will immediately compress any remaining data stored within LZ4F_cctx. + * The result of the function is the number of bytes written into dstBuffer. + * It can be zero, this means there was no data left within LZ4F_cctx. * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * LZ4F_compressOptions_t* is optional. NULL is a valid argument. */ -size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr) +size_t LZ4F_flush(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* compressOptionsPtr) { BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; @@ -916,26 +930,32 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); - if (dstCapacity < (cctxPtr->tmpInSize + 4)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); /* +4 : block header(4) */ + if (dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize)) + return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); (void)compressOptionsPtr; /* not yet useful */ /* select compression function */ compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); /* compress tmp buffer */ - dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, + dstPtr += LZ4F_makeBlock(dstPtr, + cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); - if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); + assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) < dstCapacity)); + + if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) + cctxPtr->tmpIn += cctxPtr->tmpInSize; cctxPtr->tmpInSize = 0; /* keep tmpIn within limits */ if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */ - int realDictSize = LZ4F_localSaveDict(cctxPtr); + int const realDictSize = LZ4F_localSaveDict(cctxPtr); cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; } - return dstPtr - dstStart; + return (size_t)(dstPtr - dstStart); } @@ -981,7 +1001,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, return err0r(LZ4F_ERROR_frameSize_wrong); } - return dstPtr - dstStart; + return (size_t)(dstPtr - dstStart); } @@ -1433,7 +1453,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* decode block header */ { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; - size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4; + size_t const crcSize = dctx->frameInfo.blockChecksumFlag * BFSize; if (nextCBlockSize==0) { /* frameEnd signal, no more block */ dctx->dStage = dstage_getSuffix; break; @@ -1453,7 +1473,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, dctx->tmpInTarget = nextCBlockSize + crcSize; dctx->dStage = dstage_getCBlock; if (dstPtr==dstEnd) { - nextSrcSizeHint = nextCBlockSize + crcSize + BHSize; + nextSrcSizeHint = BHSize + nextCBlockSize + crcSize; doAnotherStage = 0; } break; diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 3324f13..5e7cb11 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -110,23 +110,43 @@ LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); /* - These functions compress data in successive blocks of any size, using previous blocks as dictionary. + These functions compress data in successive blocks of any size, + using previous blocks as dictionary, to improve compression ratio. One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. There is an exception for ring buffers, which can be smaller than 64 KB. - Ring buffers scenario is automatically detected and handled by LZ4_compress_HC_continue(). + Ring-buffer scenario is automatically detected and handled within LZ4_compress_HC_continue(). - Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). - A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). + Before starting compression, state must be allocated and properly initialized. + LZ4_createStreamHC() does both, though compression level is set to LZ4HC_CLEVEL_DEFAULT. - Then, use LZ4_compress_HC_continue() to compress each successive block. - Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. - 'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), to ensure operation success. - In case of failure, the API does not guarantee context recovery, therefore context must be reset, using `LZ4_resetStreamHC()`. - If `dst` buffer size cannot be made >= LZ4_compressBound(), consider using LZ4_compress_HC_continue_destSize() instead. + Selecting the compression level can be done with LZ4_resetStreamHC_fast() (starts a new stream) + or LZ4_setCompressionLevel() (anytime, between blocks in the same stream) (experimental). + LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once. - If, for any reason, previous data block can't be preserved unmodified in memory for next compression block, - you can save it to a more stable memory space, using LZ4_saveDictHC(). - Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'. + If state space is provided manually with no guarantee of its content, for example allocated on stack, + it must be fully initialized, using LZ4_resetStreamHC(). + LZ4_resetStreamHC() is heavier, and it's guaranteed to succeed on any valid memory segment. + In contrast, LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once. + + After reset, a first "fictional block" can be designated as initial dictionary, + using LZ4_loadDictHC() (Optional). + + Invoke LZ4_compress_HC_continue() to compress each successive block. + The number of blocks is unlimited. + Previous input blocks (including initial dictionary when present) must remain accessible and unmodified during compression. + + 'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), ensuring compression success. + In case of failure, the API does not guarantee recovery, so the state _must_ be reset. + Whenever `dst` buffer size cannot be made >= LZ4_compressBound(), + consider using LZ4_compress_HC_continue_destSize() to ensure success. + + Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks, + it's possible to copy the last blocks into a more stable memory space, using LZ4_saveDictHC(). + Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer' (<= 64 KB) + + After completing a streaming compression, + it's possible to start a new stream and re-use the LZ4_streamHC_t state + by resetting it, using LZ4_resetStreamHC_fast(). */ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); @@ -136,21 +156,22 @@ LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); -LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); - /*! LZ4_compress_HC_continue_destSize() : v1.9.0+ - * Similar as LZ4_compress_HC_continue(), + * Similar to LZ4_compress_HC_continue(), * but will read as much data as possible from `src` * to fit into `targetDstSize` budget. * Result is provided into 2 parts : * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) * or 0 if compression fails. - * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`. + * `srcSizePtr` : on success, *srcSizePtr will be updated to indicate how much bytes were read from `src`. + * Note that this function may not consume the entire input. */ LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDstSize); +LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); + /*^********************************************** @@ -293,14 +314,16 @@ extern "C" { #endif /*! LZ4_setCompressionLevel() : v1.8.0+ (experimental) - * It's possible to change compression level between 2 invocations of LZ4_compress_HC_continue*() + * It's possible to change compression level + * between successive invocations of LZ4_compress_HC_continue*() + * for dynamic adaptation. */ LZ4LIB_STATIC_API void LZ4_setCompressionLevel( LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); /*! LZ4_favorDecompressionSpeed() : v1.8.2+ (experimental) - * Parser will select decisions favoring decompression over compression ratio. - * Only work at highest compression settings (level >= LZ4HC_CLEVEL_OPT_MIN) + * Opt. Parser will favor decompression speed over compression ratio. + * Only applicable to levels >= LZ4HC_CLEVEL_OPT_MIN. */ LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed( LZ4_streamHC_t* LZ4_streamHCPtr, int favor); -- cgit v0.12