From cedf9cd57f142327cb3b92a574125f46975d7a48 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 7 Sep 2022 22:50:28 -0700 Subject: allocation optimization for lz4frame compression as noted by @yixiutt, the temporary buffer inside lz4frame compression is being invalidated at end of compression, forcing it to be re-allocated at next compression job. This shouldn't be necessary. This change was introduced in #236, as a way to fix #232, but neither the issue is explained, nor why the patch fixes it. This patch revert to previous behavior, where temporary buffer is kept between compression calls. This results in a net reduction of allocation workload. Additionally, the temporary buffer should only need malloc(), not calloc(), thus saving initialization. This diff implements both changes. Long fuzzer tests will be run to ensure that no sanitizer warning get triggered. Additionally : fixed a minor ubsan warning in LZ4F_decompress(). --- lib/lz4frame.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 174f9ae..b4caff4 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -264,7 +264,7 @@ typedef struct LZ4F_cctx_s LZ4F_CustomMem cmem; LZ4F_preferences_t prefs; U32 version; - U32 cStage; + U32 cStage; /* 0 : compression uninitialized ; 1 : initialized, can compress */ const LZ4F_CDict* cdict; size_t maxBlockSize; size_t maxBufferSize; @@ -732,7 +732,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, if (cctxPtr->maxBufferSize < requiredBuffSize) { cctxPtr->maxBufferSize = 0; LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem); - cctxPtr->tmpBuff = (BYTE*)LZ4F_calloc(requiredBuffSize, cctxPtr->cmem); + cctxPtr->tmpBuff = (BYTE*)LZ4F_malloc(requiredBuffSize, cctxPtr->cmem); RETURN_ERROR_IF(cctxPtr->tmpBuff == NULL, allocation_failed); cctxPtr->maxBufferSize = requiredBuffSize; } } @@ -1176,7 +1176,6 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, } cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ - cctxPtr->maxBufferSize = 0; /* reuse HC context */ if (cctxPtr->prefs.frameInfo.contentSize) { if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) @@ -1722,10 +1721,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* history management (linked blocks only)*/ if (dctx->frameInfo.blockMode == LZ4F_blockLinked) { LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0); - } } - - srcPtr += sizeToCopy; - dstPtr += sizeToCopy; + } + srcPtr += sizeToCopy; + dstPtr += sizeToCopy; + } if (sizeToCopy == dctx->tmpInTarget) { /* all done */ if (dctx->frameInfo.blockChecksumFlag) { dctx->tmpInSize = 0; -- cgit v0.12 From 28228258450a0766697ead632c8cce0c85ed6b88 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 7 Sep 2022 23:51:30 -0700 Subject: added LZ4F_compressUpdate() in fullbench --- lib/lz4frame.h | 2 +- tests/fullbench.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 1bdf6c4..8d9380b 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -278,7 +278,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. - * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. + * `prefsPtr` is optional : NULL can be provided to set all preferences to default. * @return : number of bytes written into dstBuffer for the header * or an error code (which can be tested using LZ4F_isError()) */ diff --git a/tests/fullbench.c b/tests/fullbench.c index 9c13996..e7a8ea6 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -371,6 +371,33 @@ static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) return (int)LZ4F_compressFrame(out, LZ4F_compressFrameBound((size_t)inSize, NULL), in, (size_t)inSize, NULL); } +LZ4F_cctx* g_cctx = NULL; +static int local_LZ4F_compress(const char* in, char* out, int inSize) +{ + /* output buffer size is assumed */ + size_t const outSize = LZ4F_compressFrameBound((size_t)inSize, NULL); + size_t cSize = 0; + assert(inSize >= 0); + if (g_cctx == NULL) { + /* create and initialize LZ4F compression context the first time */ + LZ4F_createCompressionContext(&g_cctx, LZ4F_VERSION); + assert(g_cctx != NULL); + } /* re-use existing compression context otherwise */ + { size_t const cbSize = LZ4F_compressBegin(g_cctx, out, outSize, NULL); + assert(!LZ4F_isError(cbSize)); + cSize += cbSize; + } + { size_t const cuSize = LZ4F_compressUpdate(g_cctx, out+cSize, outSize-cSize, in, (size_t)inSize, NULL); + assert(!LZ4F_isError(cuSize)); + cSize += cuSize; + } + { size_t const ceSize = LZ4F_compressEnd(g_cctx, out+cSize, outSize-cSize, NULL); + assert(!LZ4F_isError(ceSize)); + cSize += ceSize; + } + return (int)cSize; +} + static LZ4F_decompressionContext_t g_dCtx; static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outSize) @@ -586,6 +613,9 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) case 30: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; chunkP[0].origSize = (int)benchedSize; nbChunks=1; break; + case 31: compressionFunction = local_LZ4F_compress; compressorName = "LZ4F_compressUpdate"; + chunkP[0].origSize = (int)benchedSize; nbChunks=1; + break; case 40: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict"; if (chunkP[0].origSize < 8) { DISPLAY(" cannot bench %s with less then 8 bytes \n", compressorName); continue; } LZ4_loadDict(&LZ4_stream, chunkP[0].origBuffer, chunkP[0].origSize); -- cgit v0.12