From bd704cf70ad7b935db0d1baaee902b28e6394bba Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 13 Sep 2014 10:08:55 +0100 Subject: lz4frame : implemented option stableSrc Improved LZ4_compressFrame() speed --- lz4frame.c | 24 +++++++++++------------- lz4frame.h | 8 ++++---- programs/frametest.c | 17 +++++++++++++---- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/lz4frame.c b/lz4frame.c index 4c63227..7290972 100644 --- a/lz4frame.c +++ b/lz4frame.c @@ -232,8 +232,9 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere */ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_preferences_t prefs = { 0 }; - LZ4F_compressionContext_t cctx = NULL; + LZ4F_cctx_internal_t cctxI = { 0 }; /* works because no allocation */ + LZ4F_preferences_t prefs = { 0 }; + LZ4F_compressOptions_t options = { 0 }; LZ4F_errorCode_t errorCode; BYTE* const dstStart = (BYTE*) dstBuffer; BYTE* dstPtr = dstStart; @@ -241,30 +242,27 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf if (preferencesPtr!=NULL) prefs = *preferencesPtr; + cctxI.version = LZ4F_VERSION; + cctxI.maxBufferSize = 64 KB; /* mess with real buffer size, to prevent allocation; works because autoflush==1 & stableSrc==1 */ prefs.autoFlush = 1; + options.stableSrc = 1; if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs)) return -ERROR_dstMaxSize_tooSmall; - errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) return errorCode; - - errorCode = LZ4F_compressBegin(cctx, dstBuffer, dstMaxSize, &prefs); /* write header */ + errorCode = LZ4F_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ if (LZ4F_isError(errorCode)) return errorCode; dstPtr += errorCode; /* header size */ dstMaxSize -= errorCode; - errorCode = LZ4F_compressUpdate(cctx, dstPtr, dstMaxSize, srcBuffer, srcSize, NULL); + errorCode = LZ4F_compressUpdate(&cctxI, dstPtr, dstMaxSize, srcBuffer, srcSize, &options); if (LZ4F_isError(errorCode)) return errorCode; dstPtr += errorCode; - errorCode = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, NULL); /* flush last block, and generate suffix */ + errorCode = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ if (LZ4F_isError(errorCode)) return errorCode; dstPtr += errorCode; - errorCode = LZ4F_freeCompressionContext(cctx); - if (LZ4F_isError(errorCode)) return errorCode; - return (dstPtr - dstStart); } @@ -496,9 +494,9 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d srcPtr += iSize; } - if ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) && (lastBlockCompressed)) + /* save last input up to 64 KB for dictionary */ + if ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) && (lastBlockCompressed) && (!compressOptionsPtr->stableSrc)) { - /* last compressed input up to 64 KB become dictionary */ if ((lastBlockCompressed==2) || ((cctxPtr->tmpBuff + cctxPtr->maxBufferSize) < (cctxPtr->tmpIn + cctxPtr->maxBlockSize))) { diff --git a/lz4frame.h b/lz4frame.h index 25cf043..d75d462 100644 --- a/lz4frame.h +++ b/lz4frame.h @@ -85,7 +85,7 @@ typedef struct { blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ - unsigned reserved[3]; + unsigned reserved[5]; } LZ4F_frameInfo_t; typedef struct { @@ -122,8 +122,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf typedef void* LZ4F_compressionContext_t; typedef struct { - unsigned stableSrc; /* unused for the time being, must be 0 */ - unsigned reserved[5]; + unsigned stableSrc; /* 1 == src content will remain available on future calls to LZ4F_compress(); avoid saving src content within tmp buffer as future dictionary */ + unsigned reserved[3]; } LZ4F_compressOptions_t; /* Resource Management */ @@ -199,7 +199,7 @@ typedef void* LZ4F_decompressionContext_t; typedef struct { unsigned stableDst; /* unused for the time being, must be 0 */ - unsigned reserved[5]; + unsigned reserved[3]; } LZ4F_decompressOptions_t; /* Resource management */ diff --git a/programs/frametest.c b/programs/frametest.c index c3662b4..73d6373 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -405,13 +405,19 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi unsigned BMId = FUZ_rand(&randState) & 1; unsigned CCflag = FUZ_rand(&randState) & 1; unsigned autoflush = (FUZ_rand(&randState) & 3) == 2; - LZ4F_preferences_t prefs = { { BSId, BMId, CCflag, 0,0,0 }, 0,autoflush, 0,0,0,0 }; + LZ4F_preferences_t prefs = { 0 }; + LZ4F_compressOptions_t options = { 0 }; unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; size_t srcSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; - result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, NULL); + options.stableSrc = ((FUZ_rand(&randState) && 3) == 2); + + result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &options); CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); op += result; ip += iSize; + if (forceFlush) { - result = LZ4F_flush(cCtx, op, oend-op, NULL); + result = LZ4F_flush(cCtx, op, oend-op, &options); CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); op += result; } } - result = LZ4F_compressEnd(cCtx, op, oend-op, NULL); + result = LZ4F_compressEnd(cCtx, op, oend-op, &options); CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result); op += result; cSize = op-(BYTE*)compressedBuffer; -- cgit v0.12