From 0400451ac22b9578274a3971ed65e6c70cd7fdb9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 10 Sep 2014 13:00:39 +0100 Subject: Fix : streaming mode bug (re-using context & buffers) --- lz4.c | 6 +++--- lz4.h | 2 +- lz4frame.c | 25 +++++++++++++------------ lz4frame.h | 1 + programs/frametest.c | 3 ++- programs/fullbench.c | 4 +--- 6 files changed, 21 insertions(+), 20 deletions(-) diff --git a/lz4.c b/lz4.c index ecb8a79..4e2026e 100644 --- a/lz4.c +++ b/lz4.c @@ -855,7 +855,7 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; - memcpy(safeBuffer, previousDictEnd - dictSize, dictSize); + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); dict->dictionary = (const BYTE*)safeBuffer; dict->dictSize = (U32)dictSize; @@ -870,9 +870,9 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) ****************************/ /* * This generic decompression function cover all use cases. - * It shall be instanciated several times, using different sets of directives + * It shall be instantiated several times, using different sets of directives * Note that it is essential this generic function is really inlined, - * in order to remove useless branches during compilation optimisation. + * in order to remove useless branches during compilation optimization. */ FORCE_INLINE int LZ4_decompress_generic( const char* source, diff --git a/lz4.h b/lz4.h index 9e192e6..44ada14 100644 --- a/lz4.h +++ b/lz4.h @@ -48,7 +48,7 @@ extern "C" { **************************************/ #define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */ #define LZ4_VERSION_MINOR 3 /* for minor interface/format changes */ -#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) int LZ4_versionNumber (void); diff --git a/lz4frame.c b/lz4frame.c index 4871d5b..071fb9e 100644 --- a/lz4frame.c +++ b/lz4frame.c @@ -145,7 +145,6 @@ typedef struct { size_t tmpOutSize; size_t tmpOutStart; XXH32_stateSpace_t xxh; - LZ4_streamDecode_t lz4ctx; BYTE header[8]; } LZ4F_dctx_internal_t; @@ -485,7 +484,7 @@ size_t LZ4F_compress(LZ4F_compressionContext_t compressionContext, void* dstBuff /* complete tmpIn block and then compress it */ BYTE* cSizePtr = dstPtr; U32 cSize; - lastBlockCompressed = 2; + lastBlockCompressed = 1; memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); srcPtr += sizeToCopy; dstPtr += 4; /* space for cSize */ @@ -717,27 +716,27 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt dctxPtr->frameInfo.blockMode = blockMode; dctxPtr->frameInfo.contentChecksumFlag = contentChecksumFlag; dctxPtr->frameInfo.blockSizeID = blockSizeID; + dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); /* init */ if (contentChecksumFlag) XXH32_resetState(&(dctxPtr->xxh), 0); - if (blockMode==blockLinked) LZ4_setStreamDecode(&(dctxPtr->lz4ctx), NULL, 0); - dctxPtr->dictSize = 0; - if (LZ4F_getBlockSize(blockSizeID) > dctxPtr->maxBufferSize) /* tmp buffers too small */ + /* alloc */ + if (dctxPtr->maxBlockSize + (dctxPtr->frameInfo.blockMode==blockLinked) > dctxPtr->maxBufferSize) /* tmp buffers too small */ { FREEMEM(dctxPtr->tmpIn); FREEMEM(dctxPtr->tmpOutBuffer); - dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); dctxPtr->maxBufferSize = dctxPtr->maxBlockSize; if (dctxPtr->frameInfo.blockMode==blockLinked) dctxPtr->maxBufferSize += 64 KB; dctxPtr->tmpIn = ALLOCATOR(dctxPtr->maxBlockSize); if (dctxPtr->tmpIn == NULL) return -ERROR_GENERIC; dctxPtr->tmpOutBuffer= ALLOCATOR(dctxPtr->maxBufferSize); if (dctxPtr->tmpOutBuffer== NULL) return -ERROR_GENERIC; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; - if (dctxPtr->frameInfo.blockMode==blockLinked) dctxPtr->tmpOut += 64 KB; - dctxPtr->dict = dctxPtr->tmpOut - dctxPtr->dictSize; } + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; + if (dctxPtr->frameInfo.blockMode==blockLinked) dctxPtr->tmpOut += 64 KB; + dctxPtr->dictSize = 0; + dctxPtr->dict = dctxPtr->tmpOut; return 7; } @@ -778,7 +777,7 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionCont } -static void LZ4F_saveDict(LZ4F_dctx_internal_t* dctxPtr, BYTE* decoded, size_t decodedSize) +static void LZ4F_saveDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* decoded, size_t decodedSize) { size_t newDictSize = decodedSize; size_t preserveDictSize; @@ -925,6 +924,8 @@ goto_decodeCBlockSize: if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; memcpy(dstPtr, srcPtr, sizeToCopy); if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy); + if (dctxPtr->frameInfo.blockMode==blockLinked) + LZ4F_saveDict(dctxPtr, srcPtr, sizeToCopy); srcPtr += sizeToCopy; dstPtr += sizeToCopy; if (sizeToCopy == dctxPtr->sizeToDecode) /* all copied */ @@ -1023,7 +1024,7 @@ goto_getSuffix: } dctxPtr->tmpInSize = 0; dctxPtr->dStage = dstage_storeSuffix; - /* break; useless, it follow */ + /* break; useless, it follows */ } case dstage_storeSuffix: { @@ -1042,7 +1043,7 @@ goto_checkSuffix: { U32 readCRC = LZ4F_readLE32(selectedIn); U32 resultCRC = XXH32_intermediateDigest(&(dctxPtr->xxh)); - if (readCRC != resultCRC) return -ERROR_GENERIC; + if (readCRC != resultCRC) return -ERROR_checksum_invalid; goodResult = OK_FrameEnd; dctxPtr->dStage = dstage_getHeader; goto _end; diff --git a/lz4frame.h b/lz4frame.h index 179271a..039ab14 100644 --- a/lz4frame.h +++ b/lz4frame.h @@ -61,6 +61,7 @@ typedef enum { OK_NoError = 0, ERROR_GENERIC = 1, ERROR_srcSize_tooLarge, ERROR_dstMaxSize_tooSmall, ERROR_allocation_failed, ERROR_compressionLevel_invalid, + ERROR_checksum_invalid, ERROR_maxCode } LZ4F_errorCodes; /* error codes are negative unsigned values. Compare function result to (-specificCode) */ diff --git a/programs/frametest.c b/programs/frametest.c index 9db2379..efe1883 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -392,9 +392,9 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi for ( ; testNb < nbTests; testNb++) { U32 randState = coreRand ^ prime1; - unsigned CCflag = FUZ_rand(&randState) & 1; unsigned BSId = 4 + (FUZ_rand(&randState) & 3); unsigned BMId = FUZ_rand(&randState) & 1; + unsigned CCflag = FUZ_rand(&randState) & 1; LZ4F_preferences_t prefs = { { BSId, BMId, CCflag, 0,0,0 }, 0,0, 0,0,0,0 }; unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; size_t srcSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; if (oSize > (size_t)(oend-op)) oSize = oend-op; + oSize = oend-op; result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); CHECK(LZ4F_isError(result), "Decompression failed (error %i)", (int)result); op += oSize; diff --git a/programs/fullbench.c b/programs/fullbench.c index 3d39458..f1d3cc1 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -60,10 +60,8 @@ #endif #include "lz4.h" -#define COMPRESSOR0 LZ4_compress #include "lz4hc.h" -#define COMPRESSOR1 LZ4_compressHC -#define DEFAULTCOMPRESSOR COMPRESSOR0 +#include "lz4frame.h" #include "xxhash.h" -- cgit v0.12