From d71b9e25b729e92871cd6a9791170e334d5199d1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 15 Sep 2014 00:59:30 +0100 Subject: small improvements to lz4frame compression --- NEWS | 4 ++ lz4frame.c | 158 +++++++++++++++++++++++++-------------------------- lz4frame.h | 2 +- programs/frametest.c | 14 +++-- 4 files changed, 90 insertions(+), 88 deletions(-) diff --git a/NEWS b/NEWS index 69e5c4e..ee4e40b 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +r123: +Added : experimental lz4frame API, thanks to Takayuki Matsuoka and Christopher Jackson for testings +Fix : test mode (-t) no longer requires confirmation, thanks to Thary Nguyen + r122: Fix : AIX & AIX64 support (SamG) Fix : mips 64-bits support (lew van) diff --git a/lz4frame.c b/lz4frame.c index 7bc2319..77b96c9 100644 --- a/lz4frame.c +++ b/lz4frame.c @@ -245,6 +245,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf cctxI.maxBufferSize = 64 KB; /* mess with real buffer size, to prevent allocation; works because autoflush==1 & stableSrc==1 */ prefs.autoFlush = 1; options.stableSrc = 1; + if (srcSize <= 64 KB) prefs.frameInfo.blockMode = blockIndependent; /* no need for linked blocks */ if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs)) return -ERROR_dstMaxSize_tooSmall; @@ -385,6 +386,27 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP } +typedef int (*compressFunc_t)(void*, const char*, char*, int, int); + +static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx) +{ + /* compress one block */ + BYTE* cSizePtr = (BYTE*)dst; + U32 cSize; + cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1)); + LZ4F_writeLE32(cSizePtr, cSize); + if (cSize == 0) /* compression failed */ + { + cSize = srcSize; + LZ4F_writeLE32(cSizePtr, srcSize + LZ4F_BLOCKUNCOMPRESSED_FLAG); + memcpy(cSizePtr+4, src, srcSize); + } + return cSize + 4; +} + + +typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; + /* LZ4F_compressUpdate() * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. @@ -403,8 +425,8 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d const BYTE* const srcEnd = srcPtr + srcSize; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; - U32 lastBlockCompressed = 0; - int (*compress)(void*, const char*, char*, int, int); + LZ4F_lastBlockStatus lastBlockCompressed = notDone; + compressFunc_t compress; if (cctxPtr->cStage != 1) return -ERROR_GENERIC; @@ -426,26 +448,17 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); srcPtr = srcEnd; cctxPtr->tmpInSize += srcSize; + /* still needs some CRC */ } else { /* complete tmpIn block and then compress it */ - BYTE* cSizePtr = dstPtr; - U32 cSize; - lastBlockCompressed = 1; + lastBlockCompressed = fromTmpBuffer; memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); srcPtr += sizeToCopy; - dstPtr += 4; /* space for cSize */ - cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)cctxPtr->tmpIn, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1)); - dstPtr += cSize; - LZ4F_writeLE32(cSizePtr, cSize); - if (cSize == 0) /* compression failed : non compressible assumed */ - { - cSize = blockSize + LZ4F_BLOCKUNCOMPRESSED_FLAG; - LZ4F_writeLE32(cSizePtr, cSize); - memcpy(dstPtr, cctxPtr->tmpIn, blockSize); - dstPtr += blockSize; - } + + dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, &(cctxPtr->lz4ctx)); + if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; } @@ -453,59 +466,45 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d while ((size_t)(srcEnd - srcPtr) >= blockSize) { - /* compress one block */ - BYTE* cSizePtr = dstPtr; - U32 cSize; - lastBlockCompressed = 2; - dstPtr += 4; /* space for cSizePtr */ - cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)srcPtr, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1)); - dstPtr += cSize; - LZ4F_writeLE32(cSizePtr, cSize); - if (cSize == 0) /* compression failed */ - { - cSize = blockSize + LZ4F_BLOCKUNCOMPRESSED_FLAG; - LZ4F_writeLE32(cSizePtr, cSize); - memcpy(dstPtr, srcPtr, blockSize); - dstPtr += blockSize; - } + /* compress full block */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, &(cctxPtr->lz4ctx)); srcPtr += blockSize; } if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { - /* compress remaining input */ - BYTE* cSizePtr = dstPtr; - U32 cSize; - size_t iSize = srcEnd - srcPtr; - lastBlockCompressed = 2; - dstPtr += 4; /* space for cSizePtr */ - cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)srcPtr, (char*)dstPtr, (int)(iSize), (int)(iSize-1)); - dstPtr += cSize; - LZ4F_writeLE32(cSizePtr, cSize); - if (cSize == 0) /* compression failed */ - { - cSize = iSize + LZ4F_BLOCKUNCOMPRESSED_FLAG; - LZ4F_writeLE32(cSizePtr, cSize); - memcpy(dstPtr, srcPtr, iSize); - dstPtr += iSize; - } - srcPtr += iSize; + /* compress remaining input < blockSize */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, &(cctxPtr->lz4ctx)); + srcPtr = srcEnd; } - /* save last input up to 64 KB for dictionary */ - if ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) && (lastBlockCompressed) && (!compressOptionsPtr->stableSrc)) + /* preserve dictionary if necessary */ + if ((cctxPtr->prefs.frameInfo.blockMode==blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { - if ((lastBlockCompressed==2) || - ((cctxPtr->tmpBuff + cctxPtr->maxBufferSize) < (cctxPtr->tmpIn + cctxPtr->maxBlockSize))) + if (compressOptionsPtr->stableSrc) { - int result; - result = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB); - if (result==0) return -ERROR_GENERIC; - cctxPtr->tmpIn = cctxPtr->tmpBuff + result; + cctxPtr->tmpIn = cctxPtr->tmpBuff; } + else + { + int realDictSize; + realDictSize = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB); + if (realDictSize==0) return -ERROR_GENERIC; + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + } + + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily blockLinked && lastBlockCompressed==fromTmpBuffer */ + { + LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB); + cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB; } - if (srcPtr < srcEnd) /* some input data left, necessarily < blockSize */ + /* some input data left, necessarily < blockSize */ + if (srcPtr < srcEnd) { /* fill tmp buffer */ size_t sizeToCopy = srcEnd - srcPtr; @@ -547,31 +546,16 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, (int(*)(void*,const char*,char*,int,int))LZ4_compress_limitedOutput_continue : LZ4_compress_limitedOutput_withState; - { - BYTE* cSizePtr = dstPtr; - U32 cSize; - dstPtr += 4; /* space for cSizePtr */ - cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)cctxPtr->tmpIn, (char*)dstPtr, (int)(cctxPtr->tmpInSize), (int)(cctxPtr->tmpInSize-1)); - dstPtr += cSize; - LZ4F_writeLE32(cSizePtr, cSize); - if (cSize == 0) /* compression failed */ - { - cSize = cctxPtr->tmpInSize + LZ4F_BLOCKUNCOMPRESSED_FLAG; - LZ4F_writeLE32(cSizePtr, cSize); - memcpy(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize); - dstPtr += cctxPtr->tmpInSize; - } - if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; - cctxPtr->tmpInSize = 0; - } + /* compress tmp buffer */ + dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, &(cctxPtr->lz4ctx)); + if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; + cctxPtr->tmpInSize = 0; - if ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) - && ((cctxPtr->tmpBuff + cctxPtr->maxBufferSize) < (cctxPtr->tmpIn + cctxPtr->maxBlockSize))) + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily blockLinked */ { - /* last 64 KB of input become dictionary */ - int result = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB); - if (!result) return ERROR_GENERIC; - cctxPtr->tmpIn = cctxPtr->tmpBuff + result; + LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB); + cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB; } return dstPtr - dstStart; @@ -1114,8 +1098,20 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, } } - if ((dctxPtr->frameInfo.blockMode==blockLinked) && (dctxPtr->dict != dctxPtr->tmpOutBuffer)) + if ( (dctxPtr->frameInfo.blockMode==blockLinked) + &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) + ) LZ4F_saveDict(dctxPtr, NULL, 0); + //(!decompressOptionsPtr->stableDst + 1) ) + /*{ + size_t newDictSize = dctxPtr->dictSize; + BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; + if ((newDictSize) > 64 KB) newDictSize = 64 KB; + memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = newDictSize; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize; + }*/ if (srcPtrsrcExpect = srcPtr; diff --git a/lz4frame.h b/lz4frame.h index 0769f6a..4bc8c59 100644 --- a/lz4frame.h +++ b/lz4frame.h @@ -187,7 +187,7 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) * The function outputs an error code if it fails (can be tested using LZ4F_isError()) * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. + * compressionContext can then be used again, starting with LZ4F_compressBegin(). */ diff --git a/programs/frametest.c b/programs/frametest.c index 4d09411..9f14bfb 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -406,7 +406,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi unsigned CCflag = FUZ_rand(&randState) & 1; unsigned autoflush = (FUZ_rand(&randState) & 7) == 2; LZ4F_preferences_t prefs = { 0 }; - LZ4F_compressOptions_t options = { 0 }; + LZ4F_compressOptions_t cOptions = { 0 }; + LZ4F_decompressOptions_t dOptions = { 0 }; unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; size_t srcSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; - options.stableSrc = ((FUZ_rand(&randState) && 3) == 2); + cOptions.stableSrc = ((FUZ_rand(&randState) && 3) == 1); - result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &options); + result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); op += result; ip += iSize; if (forceFlush) { - result = LZ4F_flush(cCtx, op, oend-op, &options); + result = LZ4F_flush(cCtx, op, oend-op, &cOptions); CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); op += result; } } - result = LZ4F_compressEnd(cCtx, op, oend-op, &options); + result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result); op += result; cSize = op-(BYTE*)compressedBuffer; @@ -471,7 +472,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi size_t oSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; if (oSize > (size_t)(oend-op)) oSize = oend-op; - result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); + dOptions.stableDst = FUZ_rand(&randState) & 1; + result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize); CHECK(LZ4F_isError(result), "Decompression failed (error %i)", (int)result); op += oSize; -- cgit v0.12