From eda8c4eb1ced10c4f6f22bca88428ca9255f2bbe Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 3 Nov 2016 18:34:54 -0700 Subject: fix #198 : no longer requires to restart streaming decompression from where it stopped. It permits relocating remaining data into another memory space. Still, the same content must be presented. --- lib/lz4frame.c | 216 ++++++++++++++++++++++++------------------------------- lib/lz4frame.h | 19 +++-- tests/.gitignore | 9 +++ 3 files changed, 110 insertions(+), 134 deletions(-) create mode 100644 tests/.gitignore diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 0423d44..fbb677c 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -171,28 +171,6 @@ typedef struct LZ4F_cctx_s U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ } LZ4F_cctx_t; -typedef struct LZ4F_dctx_s -{ - LZ4F_frameInfo_t frameInfo; - U32 version; - U32 dStage; - U64 frameRemainingSize; - size_t maxBlockSize; - size_t maxBufferSize; - const BYTE* srcExpect; - BYTE* tmpIn; - size_t tmpInSize; - size_t tmpInTarget; - BYTE* tmpOutBuffer; - const BYTE* dict; - size_t dictSize; - BYTE* tmpOut; - size_t tmpOutSize; - size_t tmpOutStart; - XXH32_state_t xxh; - BYTE header[16]; -} LZ4F_dctx_t; - /*-************************************ * Error management @@ -232,7 +210,7 @@ static size_t LZ4F_getBlockSize(unsigned blockSizeID) static BYTE LZ4F_headerChecksum (const void* header, size_t length) { - U32 xxh = XXH32(header, length, 0); + U32 const xxh = XXH32(header, length, 0); return (BYTE)(xxh >> 8); } @@ -283,16 +261,15 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere * The result of the function is the number of bytes written into dstBuffer. * The function outputs an error code if it fails (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_cctx_t cctxI; LZ4_stream_t lz4ctx; LZ4F_preferences_t prefs; LZ4F_compressOptions_t options; - LZ4F_errorCode_t errorCode; BYTE* const dstStart = (BYTE*) dstBuffer; BYTE* dstPtr = dstStart; - BYTE* const dstEnd = dstStart + dstMaxSize; + BYTE* const dstEnd = dstStart + dstCapacity; memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */ memset(&options, 0, sizeof(options)); @@ -319,22 +296,22 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf options.stableSrc = 1; - if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs)) + if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; - errorCode = LZ4F_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ - if (LZ4F_isError(errorCode)) return errorCode; - dstPtr += errorCode; /* header size */ + { size_t const headerSize = LZ4F_compressBegin(&cctxI, dstBuffer, dstCapacity, &prefs); /* write header */ + if (LZ4F_isError(headerSize)) return headerSize; + dstPtr += headerSize; /* header size */ } - errorCode = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); - if (LZ4F_isError(errorCode)) return errorCode; - dstPtr += errorCode; + { size_t const cSize = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); + if (LZ4F_isError(cSize)) return cSize; + dstPtr += cSize; } - errorCode = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ - if (LZ4F_isError(errorCode)) return errorCode; - dstPtr += errorCode; + { size_t const tailSize = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ + if (LZ4F_isError(tailSize)) return tailSize; + dstPtr += tailSize; } - if (prefs.compressionLevel >= LZ4HC_MIN_CLEVEL) /* no allocation necessary with lz4 fast */ + if (prefs.compressionLevel >= LZ4HC_MIN_CLEVEL) /* no allocation done with lz4 fast */ FREEMEM(cctxI.lz4CtxPtr); return (dstPtr - dstStart); @@ -345,19 +322,17 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf * Advanced compression functions ***********************************/ -/* LZ4F_createCompressionContext() : -* The first thing to do is to create a compressionContext object, which will be used in all compression operations. -* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. -* The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. -* The function will provide a pointer to an allocated LZ4F_compressionContext_t object. -* If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. -* Object can release its memory using LZ4F_freeCompressionContext(); -*/ +/*! LZ4F_createCompressionContext() : + * The first thing to do is to create a compressionContext object, which will be used in all compression operations. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. + * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. + * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. + * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. + * Object can release its memory using LZ4F_freeCompressionContext(); + */ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) { - LZ4F_cctx_t* cctxPtr; - - cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t)); + LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t)); if (cctxPtr==NULL) return (LZ4F_errorCode_t)(-LZ4F_ERROR_allocation_failed); cctxPtr->version = version; @@ -371,7 +346,7 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_c LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext) { - LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext; + LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext; if (cctxPtr != NULL) { /* null pointers can be safely provided to this function, like free() */ FREEMEM(cctxPtr->lz4CtxPtr); @@ -384,15 +359,14 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp /*! LZ4F_compressBegin() : -* will write the frame header into dstBuffer. -* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. -* The result of the function is the number of bytes written into dstBuffer for the header -* or an error code (can be tested using LZ4F_isError()) -*/ -size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) + * will write the frame header into dstBuffer. + * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. + * @return : number of bytes written into dstBuffer for the header + * or an error code (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefNull; - LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; BYTE* headerStart; @@ -475,13 +449,13 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP LZ4F_preferences_t prefsNull; memset(&prefsNull, 0, sizeof(prefsNull)); prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ - { const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - LZ4F_blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; - size_t blockSize = LZ4F_getBlockSize(bid); - unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; - size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; - size_t blockInfo = 4; /* default, without block CRC option */ - size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; + size_t const blockSize = LZ4F_getBlockSize(bid); + unsigned const nbBlocks = (unsigned)(srcSize / blockSize) + 1; + size_t const lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; + size_t const blockInfo = 4; /* default, without block CRC option */ + size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; } @@ -493,9 +467,8 @@ typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level) { /* compress one block */ - BYTE* cSizePtr = (BYTE*)dst; - U32 cSize; - cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); + BYTE* const cSizePtr = (BYTE*)dst; + U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); LZ4F_writeLE32(cSizePtr, cSize); if (cSize == 0) { /* compression failed */ cSize = (U32)srcSize; @@ -552,11 +525,10 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. * The function outputs an error code if it fails (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) +size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) { LZ4F_compressOptions_t cOptionsNull; - LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; - size_t blockSize = cctxPtr->maxBlockSize; + size_t const blockSize = cctxPtr->maxBlockSize; const BYTE* srcPtr = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcPtr + srcSize; BYTE* const dstStart = (BYTE*)dstBuffer; @@ -575,7 +547,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d /* complete tmp buffer */ if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ - size_t sizeToCopy = blockSize - cctxPtr->tmpInSize; + size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize; if (sizeToCopy > srcSize) { /* add src to tmpIn buffer */ memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); @@ -631,7 +603,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d /* some input data left, necessarily < blockSize */ if (srcPtr < srcEnd) { /* fill tmp buffer */ - size_t sizeToCopy = srcEnd - srcPtr; + size_t const sizeToCopy = srcEnd - srcPtr; memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); cctxPtr->tmpInSize = sizeToCopy; } @@ -652,14 +624,12 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d * The function outputs an error code if it fails (can be tested using LZ4F_isError()) * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. */ -size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; compressFunc_t compress; - if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ if (cctxPtr->cStage != 1) return (size_t)-LZ4F_ERROR_GENERIC; if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */ @@ -692,22 +662,20 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. */ -size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; - size_t errorCode; - errorCode = LZ4F_flush(compressionContext, dstBuffer, dstMaxSize, compressOptionsPtr); - if (LZ4F_isError(errorCode)) return errorCode; - dstPtr += errorCode; + size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr); + if (LZ4F_isError(flushSize)) return flushSize; + dstPtr += flushSize; LZ4F_writeLE32(dstPtr, 0); dstPtr+=4; /* endMark */ if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) { - U32 xxh = XXH32_digest(&(cctxPtr->xxh)); + U32 const xxh = XXH32_digest(&(cctxPtr->xxh)); LZ4F_writeLE32(dstPtr, xxh); dstPtr+=4; /* content Checksum */ } @@ -728,7 +696,26 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB * Frame Decompression *****************************************************/ -/* Resource management */ +struct LZ4F_dctx_s { + LZ4F_frameInfo_t frameInfo; + U32 version; + U32 dStage; + U64 frameRemainingSize; + size_t maxBlockSize; + size_t maxBufferSize; + BYTE* tmpIn; + size_t tmpInSize; + size_t tmpInTarget; + BYTE* tmpOutBuffer; + const BYTE* dict; + size_t dictSize; + BYTE* tmpOut; + size_t tmpOutSize; + size_t tmpOutStart; + XXH32_state_t xxh; + BYTE header[16]; +}; /* typedef'd to LZ4F_dctx in lz4frame.h */ + /*! LZ4F_createDecompressionContext() : * Create a decompressionContext object, which will track all decompression operations. @@ -736,20 +723,19 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB * Object can later be released using LZ4F_freeDecompressionContext(). * @return : if != 0, there was an error during context creation. */ -LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber) +LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) { - LZ4F_dctx_t* const dctxPtr = (LZ4F_dctx_t*)ALLOCATOR(sizeof(LZ4F_dctx_t)); + LZ4F_dctx* const dctxPtr = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx)); if (dctxPtr==NULL) return (LZ4F_errorCode_t)-LZ4F_ERROR_GENERIC; dctxPtr->version = versionNumber; - *LZ4F_decompressionContextPtr = (LZ4F_decompressionContext_t)dctxPtr; + *LZ4F_decompressionContextPtr = dctxPtr; return LZ4F_OK_NoError; } -LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t LZ4F_decompressionContext) +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctxPtr) { LZ4F_errorCode_t result = LZ4F_OK_NoError; - LZ4F_dctx_t* const dctxPtr = (LZ4F_dctx_t*)LZ4F_decompressionContext; if (dctxPtr != NULL) { /* can accept NULL input, like free() */ result = (LZ4F_errorCode_t)dctxPtr->dStage; FREEMEM(dctxPtr->tmpIn); @@ -760,9 +746,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t LZ4F_ } -/* ******************************************************************** */ -/* ********************* Decompression ******************************** */ -/* ******************************************************************** */ +/*==--- Streaming Decompression operations ---==*/ typedef enum { dstage_getHeader=0, dstage_storeHeader, dstage_getCBlockSize, dstage_storeCBlockSize, @@ -800,20 +784,20 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize) /*! LZ4F_decodeHeader() : - input : `srcVoidPtr` points at the **beginning of the frame** + input : `src` points at the **beginning of the frame** output : set internal values of dctx, such as dctxPtr->frameInfo and dctxPtr->dStage. Also allocates internal buffers. @return : nb Bytes read from srcVoidPtr (necessarily <= srcSize) or an error code (testable with LZ4F_isError()) */ -static size_t LZ4F_decodeHeader(LZ4F_dctx_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) +static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize) { BYTE FLG, BD, HC; unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID; size_t bufferNeeded; size_t frameHeaderSize; - const BYTE* srcPtr = (const BYTE*)srcVoidPtr; + const BYTE* srcPtr = (const BYTE*)src; /* need to decode header to get frameInfo */ if (srcSize < minFHSize) return (size_t)-LZ4F_ERROR_frameHeader_incomplete; /* minimal frame header size */ @@ -822,7 +806,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_t* dctxPtr, const void* srcVoidPtr, si /* special case : skippable frames */ if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) { dctxPtr->frameInfo.frameType = LZ4F_skippableFrame; - if (srcVoidPtr == (void*)(dctxPtr->header)) { + if (src == (void*)(dctxPtr->header)) { dctxPtr->tmpInSize = srcSize; dctxPtr->tmpInTarget = 8; dctxPtr->dStage = dstage_storeSFrameSize; @@ -910,26 +894,23 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_t* dctxPtr, const void* srcVoidPtr, si /*! LZ4F_getFrameInfo() : -* Decodes frame header information, such as blockSize. -* It is optional : you could start by calling directly LZ4F_decompress() instead. +* Decodes frame header information, such as blockSize. Usage is optional. * The objective is to extract header information without starting decompression, typically for allocation purposes. * LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t. -* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). -* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) +* The number of bytes consumed from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). +* Decompression must resume from where it stopped (srcBuffer + *srcSizePtr) * @return : hint of the better `srcSize` to use for next call to LZ4F_decompress, * or an error code which can be tested using LZ4F_isError(). */ -LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t dCtx, LZ4F_frameInfo_t* frameInfoPtr, +LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr) { - LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)dCtx; - if (dctxPtr->dStage > dstage_storeHeader) { /* note : requires dstage_* header related to be at beginning of enum */ /* frameInfo already decoded */ size_t o=0, i=0; *srcSizePtr = 0; *frameInfoPtr = dctxPtr->frameInfo; - return LZ4F_decompress(dCtx, NULL, &o, NULL, &i, NULL); /* returns : recommended nb of bytes for LZ4F_decompress() */ + return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL); /* returns : recommended nb of bytes for LZ4F_decompress() */ } else { size_t nextSrcSize, o=0; size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr); @@ -937,7 +918,7 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t dCtx, LZ4F_frameI if (*srcSizePtr < hSize) { *srcSizePtr=0; return (size_t)-LZ4F_ERROR_frameHeader_incomplete; } *srcSizePtr = hSize; - nextSrcSize = LZ4F_decompress(dCtx, NULL, &o, srcBuffer, srcSizePtr, NULL); + nextSrcSize = LZ4F_decompress(dctxPtr, NULL, &o, srcBuffer, srcSizePtr, NULL); if (dctxPtr->dStage <= dstage_storeHeader) return (size_t)-LZ4F_ERROR_frameHeader_incomplete; /* should not happen, already checked */ *frameInfoPtr = dctxPtr->frameInfo; return nextSrcSize; @@ -953,7 +934,7 @@ static int LZ4F_decompress_safe (const char* source, char* dest, int compressedS } -static void LZ4F_updateDict(LZ4F_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) +static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) { if (dctxPtr->dictSize==0) dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ @@ -1014,27 +995,26 @@ static void LZ4F_updateDict(LZ4F_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dst /*! LZ4F_decompress() : * Call this function repetitively to regenerate data compressed within srcBuffer. -* The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr. +* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. * * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). * * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. -* You will have to call it again, continuing from where it stopped. +* Remaining data will have to be presented again in a subsequent invocation. * * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. * Basically, it's the size of the current (or remaining) compressed block + header of next block. * Respecting the hint provides some boost to performance, since it allows less buffer shuffling. -* Note that this is just a hint, you can always provide any srcSize you want. -* When a frame is fully decoded, the function result will be 0. -* If decompression failed, function result is an error code which can be tested using LZ4F_isError(). +* Note that this is just a hint, it's always possible to any srcSize value. +* When a frame is fully decoded, @return will be 0. +* If decompression failed, @return is an error code which can be tested using LZ4F_isError(). */ -size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, +size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, const LZ4F_decompressOptions_t* decompressOptionsPtr) { - LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)decompressionContext; LZ4F_decompressOptions_t optionsNull; const BYTE* const srcStart = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcStart + *srcSizePtr; @@ -1052,11 +1032,6 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, *srcSizePtr = 0; *dstSizePtr = 0; - /* expect to continue decoding src buffer where it left previously */ - if (dctxPtr->srcExpect != NULL) { - if (srcStart != dctxPtr->srcExpect) return (size_t)-LZ4F_ERROR_srcPtr_wrong; - } - /* programmed as a state machine */ while (doAnotherStage) { @@ -1105,8 +1080,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, if (dctxPtr->dStage == dstage_storeCBlockSize) /* can be skipped */ case dstage_storeCBlockSize: - { - size_t sizeToCopy = BHSize - dctxPtr->tmpInSize; + { size_t sizeToCopy = BHSize - dctxPtr->tmpInSize; if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; @@ -1402,12 +1376,6 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, } } - /* require function to be called again from position where it stopped */ - if (srcPtrsrcExpect = srcPtr; - else - dctxPtr->srcExpect = NULL; - *srcSizePtr = (srcPtr - srcStart); *dstSizePtr = (dstPtr - dstStart); return nextSrcSizeHint; diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 2dd3818..63abc60 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -276,13 +276,13 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); * This function decodes frame header information (such as max blockSize, frame checksum, etc.). * Its usage is optional. The objective is to extract frame header information, typically for allocation purposes. * A header size is variable and can length from 7 to 15 bytes. It's possible to provide more input bytes than that. - * The number of bytes read from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - * (note that LZ4F_getFrameInfo() can also be used anytime *after* decompression is started, in which case 0 input byte is enough) + * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + * Decompression must resume from this point (srcBuffer + *srcSizePtr). + * Note that LZ4F_getFrameInfo() can also be used anytime *after* decompression is started, in which case 0 input byte can be enough. * Frame header info is *copied into* an already allocated LZ4F_frameInfo_t structure. * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, * or an error code which can be tested using LZ4F_isError() * (typically, when there is not enough src bytes to fully decode the frame header) - * Decompression is expected to resume from where it stopped (srcBuffer + *srcSizePtr) */ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr, @@ -297,18 +297,17 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). * Number of bytes read can be < number of bytes provided, meaning there is some more data to decode. * It typically happens when dstBuffer is not large enough to contain all decoded data. - * In which case, LZ4F_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr) - * The function will check this condition, and refuse to continue if it is not respected. + * Remaining data will have to be presented again in a subsequent invocation. * - * `dstBuffer` is expected to be flushed between each call to the function, its content will be overwritten. - * `dst` arguments can be changed at will at each consecutive call to the function. + * `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten. + * `dstBuffer` can be changed at will between each consecutive function invocation. * - * The function result is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. + * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. * Schematically, it's the size of the current (or remaining) compressed block + header of next block. * Respecting the hint provides some boost to performance, since it does skip intermediate buffers. * This is just a hint though, it's always possible to provide any srcSize. - * When a frame is fully decoded, the function result will be 0 (no more data expected). - * If decompression failed, function result is an error code, which can be tested using LZ4F_isError(). + * When a frame is fully decoded, @return will be 0 (no more data expected). + * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). * * After a frame is fully decoded, dctx can be used again to decompress another frame. */ diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..8861e67 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,9 @@ + +# test build artefacts +datagen +frametest +frametest32 +fullbench +fullbench32 +fuzzer +fuzzer32 -- cgit v0.12