From 7cf0bb97b2a988cb17435780d19e145147dd9f70 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 14:18:01 +0100 Subject: LZ4F_compressBound(0) provides upper bound for LZ4F_flush() and LZ4F_compressEnd() [#290, suggested by @vtermanis] --- lib/lz4frame.c | 15 +++++++++++---- lib/lz4frame.h | 23 ++++++++++++----------- tests/frametest.c | 7 +++++++ 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 3c2b788..626ac98 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -158,7 +158,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB static const size_t minFHSize = 7; -static const size_t maxFHSize = 15; /* max Frame Header Size */ +static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 15 */ static const size_t BHSize = 4; @@ -254,20 +254,27 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI return requestedBSID; } +/* LZ4F_compressBound() : + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + */ static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered) { LZ4F_preferences_t prefsNull; memset(&prefsNull, 0, sizeof(prefsNull)); prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + U32 const flush = prefsPtr->autoFlush | (srcSize==0); LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; size_t const blockSize = LZ4F_getBlockSize(bid); size_t const maxBuffered = blockSize - 1; size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); size_t const maxSrcSize = srcSize + bufferedSize; unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); - size_t const partialBlockSize = srcSize & (blockSize-1); - size_t const lastBlockSize = prefsPtr->autoFlush ? partialBlockSize : 0; + size_t const partialBlockSize = (srcSize - (srcSize==0)) & (blockSize-1); /* 0 => -1 == MAX => blockSize-1 */ + size_t const lastBlockSize = flush ? partialBlockSize : 0; unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */ @@ -398,7 +405,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. + * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. * @return : number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 3104d2e..a4a4ce6 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -206,10 +206,11 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /* Compression */ +#define LZ4F_HEADER_SIZE_MAX 15 LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr); /* LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header. Maximum header size is 15 bytes. + * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. * @return : number of bytes written into dstBuffer for the header * or an error code (which can be tested using LZ4F_isError()) @@ -217,20 +218,20 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t d LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); /* LZ4F_compressBound() : - * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. - * Different preferences can produce different results. - * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. - * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. */ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); /* LZ4F_compressUpdate() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * An important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. - * This value is provided by using LZ4F_compressBound(). + * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. + * This value is provided by LZ4F_compressBound(). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). * LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) */ @@ -245,12 +246,12 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapaci * or an error code if it fails (which can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr); +LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); /* LZ4F_compressEnd() : - * To properly finish the compressed frame, invoke LZ4F_compressEnd(). + * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). * It will flush whatever data remained within `cctx` (like LZ4_flush()) * and properly finalize the frame, with an endMark and a checksum. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) * or an error code if it fails (which can be tested using LZ4F_isError()) * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. diff --git a/tests/frametest.c b/tests/frametest.c index a99728f..fd07377 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -204,6 +204,13 @@ int basicTests(U32 seed, double compressibility) FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + /* LZ4F_compressBound() : special case : srcSize == 0 */ + DISPLAYLEVEL(3, "LZ4F_compressBound(0) = "); + { size_t const cBound = LZ4F_compressBound(0, NULL); + if (cBound < 64 KB) goto _output_error; + DISPLAYLEVEL(3, " %u \n", (U32)cBound); + } + /* Special case : null-content frame */ testSize = 0; DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n"); -- cgit v0.12