diff options
author | Yann Collet <cyan@fb.com> | 2017-08-15 23:57:00 (GMT) |
---|---|---|
committer | Yann Collet <cyan@fb.com> | 2017-08-15 23:57:00 (GMT) |
commit | a1f3a27e9b3caf42d31100f96bbb88866e08a517 (patch) | |
tree | 014922fceee22f63d46a0d04328234c3711f4799 /lib | |
parent | 01cdbfb5feda4adee65cc3da04b426c0c22368a3 (diff) | |
parent | af9d72b7f6809ec972787c446d464a8f9be443ba (diff) | |
download | lz4-a1f3a27e9b3caf42d31100f96bbb88866e08a517.zip lz4-a1f3a27e9b3caf42d31100f96bbb88866e08a517.tar.gz lz4-a1f3a27e9b3caf42d31100f96bbb88866e08a517.tar.bz2 |
Merge branch 'dev' into installVars and fixed conflicts
Diffstat (limited to 'lib')
-rw-r--r-- | lib/lz4.c | 7 | ||||
-rw-r--r-- | lib/lz4.h | 38 | ||||
-rw-r--r-- | lib/lz4frame.c | 1063 | ||||
-rw-r--r-- | lib/lz4frame.h | 132 | ||||
-rw-r--r-- | lib/lz4frame_static.h | 56 | ||||
-rw-r--r-- | lib/lz4hc.c | 24 | ||||
-rw-r--r-- | lib/lz4hc.h | 23 |
7 files changed, 817 insertions, 526 deletions
@@ -945,6 +945,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { + if (!LZ4_stream) return 0; /* support free on NULL */ FREEMEM(LZ4_stream); return (0); } @@ -1284,11 +1285,6 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize) /*===== streaming decompression functions =====*/ -/* - * If you prefer dynamic allocation methods, - * LZ4_createStreamDecode() - * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure. - */ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); @@ -1297,6 +1293,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void) int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { + if (!LZ4_stream) return 0; /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } @@ -259,8 +259,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in * Important : Previous data blocks are assumed to remain present and unmodified ! * 'dst' buffer must be already allocated. * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. - * After an error, the stream status is invalid, and it can only be reset or freed. + * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0. + * After an error, the stream status is invalid, it can only be reset or freed. */ LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -291,22 +291,21 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str */ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); -/*! -LZ4_decompress_*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - In the case of a ring buffers, decoding buffer must be either : - - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including small ones ( < 64 KB). - - _At least_ 64 KB + 8 bytes + maxBlockSize. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including larger than decoding buffer. - Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - and indicate where it is saved using LZ4_setStreamDecode() +/*! LZ4_decompress_*_continue() : + * These decoding functions allow decompression of multiple blocks in "streaming" mode. + * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) + * In the case of a ring buffers, decoding buffer must be either : + * - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + * In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + * maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including small ones ( < 64 KB). + * - _At least_ 64 KB + 8 bytes + maxBlockSize. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including larger than decoding buffer. + * Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + * and indicate where it is saved using LZ4_setStreamDecode() */ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); @@ -380,7 +379,7 @@ typedef struct { * init this structure before first use. * note : only use in association with static linking ! * this definition is not API/ABI safe, - * and may change in a future version ! + * it may change in a future version ! */ #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) @@ -458,6 +457,7 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4 #endif /* LZ4_H_2983827168210 */ + #if defined (__cplusplus) } #endif diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 994cc8b..e613901 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -61,13 +61,14 @@ You can contact the author at : **************************************/ #include "lz4frame_static.h" #include "lz4.h" +#define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" /*-************************************ -* Common Utils +* Debug **************************************/ #define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ @@ -158,7 +159,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB static const size_t minFHSize = 7; -static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 15 */ +static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ static const size_t BHSize = 4; @@ -170,6 +171,7 @@ typedef struct LZ4F_cctx_s LZ4F_preferences_t prefs; U32 version; U32 cStage; + const LZ4F_CDict* cdict; size_t maxBlockSize; size_t maxBufferSize; BYTE* tmpBuff; @@ -178,7 +180,7 @@ typedef struct LZ4F_cctx_s U64 totalInSize; XXH32_state_t xxh; void* lz4CtxPtr; - U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ + U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ } LZ4F_cctx_t; @@ -216,6 +218,8 @@ static LZ4F_errorCode_t err0r(LZ4F_errorCodes code) unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } +int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; } + /*-************************************ * Private functions @@ -256,11 +260,11 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI return requestedBSID; } -/* LZ4F_compressBound() : - * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. - * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario. - * Result is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers. - * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. +/*! LZ4F_compressBound_internal() : + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario. + * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. */ static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, @@ -271,8 +275,8 @@ static size_t LZ4F_compressBound_internal(size_t srcSize, prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; U32 const flush = prefsPtr->autoFlush | (srcSize==0); - LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; - size_t const blockSize = LZ4F_getBlockSize(bid); + LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID; + size_t const blockSize = LZ4F_getBlockSize(blockID); size_t const maxBuffered = blockSize - 1; size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); size_t const maxSrcSize = srcSize + bufferedSize; @@ -281,17 +285,19 @@ static size_t LZ4F_compressBound_internal(size_t srcSize, size_t const lastBlockSize = flush ? partialBlockSize : 0; unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); - size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */ + size_t const blockHeaderSize = 4; + size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag; size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); - return (blockHeaderSize * nbBlocks) + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;; + return ((blockHeaderSize + blockCRCSize) * nbBlocks) + + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd; } } size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefs; - size_t const headerSize = maxFHSize; /* max header size, including magic number and frame content size */ + size_t const headerSize = maxFHSize; /* max header size, including optional fields */ if (preferencesPtr!=NULL) prefs = *preferencesPtr; else memset(&prefs, 0, sizeof(prefs)); @@ -301,16 +307,19 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere } -/*! LZ4F_compressFrame() : - * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step. - * The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. - * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) - * Get the minimum value of dstCapacity by using LZ4F_compressFrameBound(). - * The LZ4F_preferences_t structure is optional : if NULL is provided as argument, preferences will be set to default. - * The result of the function is the number of bytes written into dstBuffer. - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) +/*! LZ4F_compressFrame_usingCDict() : + * Compress srcBuffer using a dictionary, in a single step. + * cdict can be NULL, in which case, no dictionary is used. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, + * however, it's the only way to provide a dictID, so it's not recommended. + * @return : number of bytes written into dstBuffer, + * or an error code if it fails (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) { LZ4F_cctx_t cctxI; LZ4_stream_t lz4ctx; @@ -321,10 +330,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu BYTE* const dstEnd = dstStart + dstCapacity; memset(&cctxI, 0, sizeof(cctxI)); - memset(&options, 0, sizeof(options)); - cctxI.version = LZ4F_VERSION; - cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works because autoflush==1 & stableSrc==1 */ + cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */ if (preferencesPtr!=NULL) prefs = *preferencesPtr; @@ -333,22 +340,23 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (prefs.frameInfo.contentSize != 0) prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - cctxI.lz4CtxPtr = &lz4ctx; - cctxI.lz4CtxLevel = 1; - } /* otherwise : will be created within LZ4F_compressBegin */ - prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) - prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* no need for linked blocks */ + prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */ + if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { + cctxI.lz4CtxPtr = &lz4ctx; + cctxI.lz4CtxLevel = 1; + } /* fast compression context pre-created on stack */ + + memset(&options, 0, sizeof(options)); options.stableSrc = 1; - if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */ + if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */ return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - { size_t const headerSize = LZ4F_compressBegin(&cctxI, dstBuffer, dstCapacity, &prefs); /* write header */ + { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs); /* write header */ if (LZ4F_isError(headerSize)) return headerSize; dstPtr += headerSize; /* header size */ } @@ -360,24 +368,91 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (LZ4F_isError(tailSize)) return tailSize; dstPtr += tailSize; } - if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* Ctx allocation only for lz4hc */ + if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* Ctx allocation only for lz4hc */ FREEMEM(cctxI.lz4CtxPtr); return (dstPtr - dstStart); } +/*! LZ4F_compressFrame() : + * Compress an entire srcBuffer into a valid LZ4 frame, in a single step. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity, + srcBuffer, srcSize, + NULL, preferencesPtr); +} + + +/*-*************************************************** +* Dictionary compression +*****************************************************/ + +struct LZ4F_CDict_s { + void* dictContent; + LZ4_stream_t* fastCtx; + LZ4_streamHC_t* HCCtx; +}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */ + +/*! LZ4F_createCDict() : + * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict + * @return : digested dictionary for compression, or NULL if failed */ +LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) +{ + const char* dictStart = (const char*)dictBuffer; + LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict)); + if (!cdict) return NULL; + if (dictSize > 64 KB) { + dictStart += dictSize - 64 KB; + dictSize = 64 KB; + } + cdict->dictContent = ALLOCATOR(dictSize); + cdict->fastCtx = LZ4_createStream(); + cdict->HCCtx = LZ4_createStreamHC(); + if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) { + LZ4F_freeCDict(cdict); + return NULL; + } + memcpy(cdict->dictContent, dictStart, dictSize); + LZ4_resetStream(cdict->fastCtx); + LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize); + LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT); + LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize); + return cdict; +} + +void LZ4F_freeCDict(LZ4F_CDict* cdict) +{ + if (cdict==NULL) return; /* support free on NULL */ + FREEMEM(cdict->dictContent); + LZ4_freeStream(cdict->fastCtx); + LZ4_freeStreamHC(cdict->HCCtx); + FREEMEM(cdict); +} + + /*-********************************* * Advanced compression functions ***********************************/ /*! LZ4F_createCompressionContext() : - * The first thing to do is to create a compressionContext object, which will be used in all compression operations. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. - * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. - * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. - * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. - * Object can release its memory using LZ4F_freeCompressionContext(); + * The first thing to do is to create a compressionContext object, which will be used in all compression operations. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. + * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries. + * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. + * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. + * Object can release its memory using LZ4F_freeCompressionContext(); */ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) { @@ -385,7 +460,7 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_c if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed); cctxPtr->version = version; - cctxPtr->cStage = 0; /* Next stage : write header */ + cctxPtr->cStage = 0; /* Next stage : init stream */ *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; @@ -397,8 +472,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp { LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext; - if (cctxPtr != NULL) { /* null pointers can be safely provided to this function, like free() */ - FREEMEM(cctxPtr->lz4CtxPtr); + if (cctxPtr != NULL) { /* support free on NULL */ + FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */ FREEMEM(cctxPtr->tmpBuff); FREEMEM(LZ4F_compressionContext); } @@ -407,22 +482,23 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp } -/*! LZ4F_compressBegin() : - * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. - * @return : number of bytes written into dstBuffer for the header - * or an error code (can be tested using LZ4F_isError()) +/*! LZ4F_compressBegin_usingCDict() : + * init streaming compression and writes frame header into dstBuffer. + * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. + * @return : number of bytes written into dstBuffer for the header + * or an error code (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr) +size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefNull; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; BYTE* headerStart; - size_t requiredBuffSize; if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - if (cctxPtr->cStage != 0) return err0r(LZ4F_ERROR_GENERIC); memset(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; cctxPtr->prefs = *preferencesPtr; @@ -437,31 +513,46 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed); cctxPtr->lz4CtxLevel = tableID; - } - } + } } /* Buffer Management */ - if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + if (cctxPtr->prefs.frameInfo.blockSizeID == 0) + cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); - requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB); - if (preferencesPtr->autoFlush) - requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB; /* just needs dict */ + { size_t const requiredBuffSize = preferencesPtr->autoFlush ? + (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB : /* only needs windows size */ + cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB); - if (cctxPtr->maxBufferSize < requiredBuffSize) { - cctxPtr->maxBufferSize = 0; - FREEMEM(cctxPtr->tmpBuff); - cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); - if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed); - cctxPtr->maxBufferSize = requiredBuffSize; - } + if (cctxPtr->maxBufferSize < requiredBuffSize) { + cctxPtr->maxBufferSize = 0; + FREEMEM(cctxPtr->tmpBuff); + cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); + if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed); + cctxPtr->maxBufferSize = requiredBuffSize; + } } cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; XXH32_reset(&(cctxPtr->xxh), 0); - if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) - LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); - else - LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); + + /* context init */ + cctxPtr->cdict = cdict; + if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) { + /* frame init only for blockLinked : blockIndependent will be init at each block */ + if (cdict) { + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { + memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx)); + } else { + memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx)); + LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + } + } else { + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) + LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); + else + LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); + } + } /* Magic Number */ LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); @@ -470,9 +561,11 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit /* FLG Byte */ *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ - + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ - + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ - + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */ + + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) + + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4) + + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3) + + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) + + (cctxPtr->prefs.frameInfo.dictID > 0) ); /* BD Byte */ *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); /* Optional Frame content size field */ @@ -481,16 +574,36 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit dstPtr += 8; cctxPtr->totalInSize = 0; } - /* CRC Byte */ + /* Optional dictionary ID field */ + if (cctxPtr->prefs.frameInfo.dictID) { + LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID); + dstPtr += 4; + } + /* Header CRC Byte */ *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart); dstPtr++; cctxPtr->cStage = 1; /* header written, now request input data block */ - return (dstPtr - dstStart); } +/*! LZ4F_compressBegin() : + * init streaming compression and writes frame header into dstBuffer. + * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. + * preferencesPtr can be NULL, in which case default parameters are selected. + * @return : number of bytes written into dstBuffer for the header + * or an error code (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressBegin_usingCDict(cctxPtr, dstBuffer, dstCapacity, + NULL, preferencesPtr); +} + + /* LZ4F_compressBound() : * @ return size of Dst buffer given a srcSize to handle worst case situations. * The LZ4F_frameInfo_t structure is optional : if NULL, preferences will be set to cover worst case situations. @@ -502,49 +615,75 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP } -typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level); +typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict); -static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level) + +/*! LZ4F_makeBlock(): + * compress a single block, add header and checksum + * assumption : dst buffer capacity is >= srcSize */ +static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, + compressFunc_t compress, void* lz4ctx, int level, + const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag) { - /* compress a single block */ BYTE* const cSizePtr = (BYTE*)dst; - U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); + U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), + (int)(srcSize), (int)(srcSize-1), + level, cdict); LZ4F_writeLE32(cSizePtr, cSize); if (cSize == 0) { /* compression failed */ cSize = (U32)srcSize; LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); memcpy(cSizePtr+4, src, srcSize); } - return cSize + 4; + if (crcFlag) { + U32 const crc32 = XXH32(cSizePtr+4, cSize, 0); /* checksum of compressed data */ + LZ4F_writeLE32(cSizePtr+4+cSize, crc32); + } + return 4 + cSize + ((U32)crcFlag)*4; } -static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level) +static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { int const acceleration = (level < -1) ? -level : 1; + if (cdict) { + memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx)); + return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); + } return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration); } -static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level) +static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { int const acceleration = (level < -1) ? -level : 1; + (void)cdict; /* init once at beginning of frame */ return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); } -static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { - (void) level; - return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize); + if (cdict) { + memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx)); + LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level); + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); + } + return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level); +} + +static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + (void)level; (void)cdict; /* init once at beginning of frame */ + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); } static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) { if (level < LZ4HC_CLEVEL_MIN) { - if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; - return LZ4F_localLZ4_compress_limitedOutput_continue; + if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock; + return LZ4F_compressBlock_continue; } - if (blockMode == LZ4F_blockIndependent) return LZ4_compress_HC_extStateHC; - return LZ4F_localLZ4_compressHC_limitedOutput_continue; + if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC; + return LZ4F_compressBlockHC_continue; } static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) @@ -557,15 +696,16 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; /*! LZ4F_compressUpdate() : -* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. -* The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. -* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) -* You can get the minimum value of dstCapacity by using LZ4F_compressBound() -* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. -* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -*/ -size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) { LZ4F_compressOptions_t cOptionsNull; size_t const blockSize = cctxPtr->maxBlockSize; @@ -597,7 +737,9 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); srcPtr += sizeToCopy; - dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; @@ -605,16 +747,20 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci } while ((size_t)(srcEnd - srcPtr) >= blockSize) { - /* compress full block */ + /* compress full blocks */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); srcPtr += blockSize; } if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { /* compress remaining input < blockSize */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); srcPtr = srcEnd; } @@ -676,7 +822,9 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); /* compress tmp buffer */ - dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; cctxPtr->tmpInSize = 0; @@ -733,10 +881,23 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, * Frame Decompression *****************************************************/ +typedef enum { + dstage_getFrameHeader=0, dstage_storeFrameHeader, + dstage_init, + dstage_getBlockHeader, dstage_storeBlockHeader, + dstage_copyDirect, dstage_getBlockChecksum, + dstage_getCBlock, dstage_storeCBlock, + dstage_decodeCBlock, dstage_decodeCBlock_intoDst, + dstage_decodeCBlock_intoTmp, dstage_flushOut, + dstage_getSuffix, dstage_storeSuffix, + dstage_getSFrameSize, dstage_storeSFrameSize, + dstage_skipSkippable +} dStage_t; + struct LZ4F_dctx_s { LZ4F_frameInfo_t frameInfo; U32 version; - U32 dStage; + dStage_t dStage; U64 frameRemainingSize; size_t maxBlockSize; size_t maxBufferSize; @@ -744,40 +905,41 @@ struct LZ4F_dctx_s { size_t tmpInSize; size_t tmpInTarget; BYTE* tmpOutBuffer; - const BYTE* dict; + const BYTE* dict; size_t dictSize; BYTE* tmpOut; size_t tmpOutSize; size_t tmpOutStart; XXH32_state_t xxh; + XXH32_state_t blockChecksum; BYTE header[16]; }; /* typedef'd to LZ4F_dctx in lz4frame.h */ /*! LZ4F_createDecompressionContext() : - * Create a decompressionContext object, which will track all decompression operations. - * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object. - * Object can later be released using LZ4F_freeDecompressionContext(). - * @return : if != 0, there was an error during context creation. + * Create a decompressionContext object, which will track all decompression operations. + * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object. + * Object can later be released using LZ4F_freeDecompressionContext(). + * @return : if != 0, there was an error during context creation. */ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) { - LZ4F_dctx* const dctxPtr = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx)); - if (dctxPtr==NULL) return err0r(LZ4F_ERROR_GENERIC); + LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx)); + if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC); - dctxPtr->version = versionNumber; - *LZ4F_decompressionContextPtr = dctxPtr; + dctx->version = versionNumber; + *LZ4F_decompressionContextPtr = dctx; return LZ4F_OK_NoError; } -LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctxPtr) +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx) { LZ4F_errorCode_t result = LZ4F_OK_NoError; - if (dctxPtr != NULL) { /* can accept NULL input, like free() */ - result = (LZ4F_errorCode_t)dctxPtr->dStage; - FREEMEM(dctxPtr->tmpIn); - FREEMEM(dctxPtr->tmpOutBuffer); - FREEMEM(dctxPtr); + if (dctx != NULL) { /* can accept NULL input, like free() */ + result = (LZ4F_errorCode_t)dctx->dStage; + FREEMEM(dctx->tmpIn); + FREEMEM(dctx->tmpOutBuffer); + FREEMEM(dctx); } return result; } @@ -785,22 +947,11 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctxPtr) /*==--- Streaming Decompression operations ---==*/ -typedef enum { - dstage_getHeader=0, dstage_storeHeader, - dstage_init, - dstage_getCBlockSize, dstage_storeCBlockSize, - dstage_copyDirect, - dstage_getCBlock, dstage_storeCBlock, - dstage_decodeCBlock, dstage_decodeCBlock_intoDst, - dstage_decodeCBlock_intoTmp, dstage_flushOut, - dstage_getSuffix, dstage_storeSuffix, - dstage_getSFrameSize, dstage_storeSFrameSize, - dstage_skipSkippable -} dStage_t; - void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) { - dctx->dStage = dstage_getHeader; + dctx->dStage = dstage_getFrameHeader; + dctx->dict = NULL; + dctx->dictSize = 0; } @@ -817,12 +968,14 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize) if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8; /* control magic number */ - if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); + if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) + return err0r(LZ4F_ERROR_frameType_unknown); /* Frame Header Size */ { BYTE const FLG = ((const BYTE*)src)[4]; U32 const contentSizeFlag = (FLG>>3) & _1BIT; - return contentSizeFlag ? maxFHSize : minFHSize; + U32 const dictIDFlag = FLG & _1BIT; + return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); } } @@ -830,62 +983,63 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize) /*! LZ4F_decodeHeader() : * input : `src` points at the **beginning of the frame** * output : set internal values of dctx, such as - * dctxPtr->frameInfo and dctxPtr->dStage. + * dctx->frameInfo and dctx->dStage. * Also allocates internal buffers. * @return : nb Bytes read from src (necessarily <= srcSize) * or an error code (testable with LZ4F_isError()) */ -static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize) +static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize) { - unsigned blockMode, contentSizeFlag, contentChecksumFlag, blockSizeID; + unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID; size_t frameHeaderSize; const BYTE* srcPtr = (const BYTE*)src; /* need to decode header to get frameInfo */ if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */ - memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); + memset(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo)); /* special case : skippable frames */ if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) { - dctxPtr->frameInfo.frameType = LZ4F_skippableFrame; - if (src == (void*)(dctxPtr->header)) { - dctxPtr->tmpInSize = srcSize; - dctxPtr->tmpInTarget = 8; - dctxPtr->dStage = dstage_storeSFrameSize; + dctx->frameInfo.frameType = LZ4F_skippableFrame; + if (src == (void*)(dctx->header)) { + dctx->tmpInSize = srcSize; + dctx->tmpInTarget = 8; + dctx->dStage = dstage_storeSFrameSize; return srcSize; } else { - dctxPtr->dStage = dstage_getSFrameSize; + dctx->dStage = dstage_getSFrameSize; return 4; } } /* control magic number */ - if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); - dctxPtr->frameInfo.frameType = LZ4F_frame; + if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) + return err0r(LZ4F_ERROR_frameType_unknown); + dctx->frameInfo.frameType = LZ4F_frame; /* Flags */ { U32 const FLG = srcPtr[4]; U32 const version = (FLG>>6) & _2BITS; - U32 const blockChecksumFlag = (FLG>>4) & _1BIT; + blockChecksumFlag = (FLG>>4) & _1BIT; blockMode = (FLG>>5) & _1BIT; contentSizeFlag = (FLG>>3) & _1BIT; contentChecksumFlag = (FLG>>2) & _1BIT; + dictIDFlag = FLG & _1BIT; /* validate */ - if (((FLG>>0)&_2BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */ + if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */ if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */ - if (blockChecksumFlag != 0) return err0r(LZ4F_ERROR_blockChecksum_unsupported); /* Not supported for the time being */ } /* Frame Header Size */ - frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize; + frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); if (srcSize < frameHeaderSize) { /* not enough input to fully decode frame header */ - if (srcPtr != dctxPtr->header) - memcpy(dctxPtr->header, srcPtr, srcSize); - dctxPtr->tmpInSize = srcSize; - dctxPtr->tmpInTarget = frameHeaderSize; - dctxPtr->dStage = dstage_storeHeader; + if (srcPtr != dctx->header) + memcpy(dctx->header, srcPtr, srcSize); + dctx->tmpInSize = srcSize; + dctx->tmpInTarget = frameHeaderSize; + dctx->dStage = dstage_storeFrameHeader; return srcSize; } @@ -898,50 +1052,56 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS } /* check header */ - { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); - if (HC != srcPtr[frameHeaderSize-1]) return err0r(LZ4F_ERROR_headerChecksum_invalid); } + { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); + if (HC != srcPtr[frameHeaderSize-1]) + return err0r(LZ4F_ERROR_headerChecksum_invalid); + } /* save */ - dctxPtr->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; - dctxPtr->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag; - dctxPtr->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; - dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); + dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; + dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag; + dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag; + dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; + dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID); if (contentSizeFlag) - dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); + dctx->frameRemainingSize = + dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); + if (dictIDFlag) + dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5); - dctxPtr->dStage = dstage_init; + dctx->dStage = dstage_init; return frameHeaderSize; } /*! LZ4F_getFrameInfo() : - * This function extracts frame parameters (max blockSize, frame checksum, etc.). - * Usage is optional. Objective is to provide relevant information for allocation purposes. - * This function works in 2 situations : + * This function extracts frame parameters (max blockSize, frame checksum, etc.). + * Usage is optional. Objective is to provide relevant information for allocation purposes. + * This function works in 2 situations : * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process. * Amount of input data provided must be large enough to successfully decode the frame header. * A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum. * - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx. - * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - * Decompression must resume from (srcBuffer + *srcSizePtr). + * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + * Decompression must resume from (srcBuffer + *srcSizePtr). * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, * or an error code which can be tested using LZ4F_isError() - * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped. - * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. + * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped. + * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. */ -LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameInfoPtr, +LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr) { - if (dctxPtr->dStage > dstage_storeHeader) { /* assumption : dstage_* header enum at beginning of range */ + if (dctx->dStage > dstage_storeFrameHeader) { /* assumption : dstage_* header enum at beginning of range */ /* frameInfo already decoded */ size_t o=0, i=0; *srcSizePtr = 0; - *frameInfoPtr = dctxPtr->frameInfo; + *frameInfoPtr = dctx->frameInfo; /* returns : recommended nb of bytes for LZ4F_decompress() */ - return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL); + return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL); } else { - if (dctxPtr->dStage == dstage_storeHeader) { + if (dctx->dStage == dstage_storeFrameHeader) { /* frame decoding already started, in the middle of header => automatic fail */ *srcSizePtr = 0; return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted); @@ -954,109 +1114,99 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn return err0r(LZ4F_ERROR_frameHeader_incomplete); } - decodeResult = LZ4F_decodeHeader(dctxPtr, srcBuffer, hSize); + decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize); if (LZ4F_isError(decodeResult)) { *srcSizePtr = 0; } else { *srcSizePtr = decodeResult; decodeResult = BHSize; /* block header size */ } - *frameInfoPtr = dctxPtr->frameInfo; + *frameInfoPtr = dctx->frameInfo; return decodeResult; } } } -/* trivial redirector, for common prototype */ -static int LZ4F_decompress_safe (const char* src, - char* dst, - int compressedSize, - int dstCapacity, - const char* dictStart, - int dictSize) +/* LZ4F_updateDict() : + * only used for LZ4F_blockLinked mode */ +static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) { - (void)dictStart; (void)dictSize; - return LZ4_decompress_safe (src, dst, compressedSize, dstCapacity); -} - + if (dctx->dictSize==0) + dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ -static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) -{ - if (dctxPtr->dictSize==0) - dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ - - if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) { /* dictionary continuity */ - dctxPtr->dictSize += dstSize; + if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity */ + dctx->dictSize += dstSize; return; } if (dstPtr - dstPtr0 + dstSize >= 64 KB) { /* dstBuffer large enough to become dictionary */ - dctxPtr->dict = (const BYTE*)dstPtr0; - dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize; + dctx->dict = (const BYTE*)dstPtr0; + dctx->dictSize = dstPtr - dstPtr0 + dstSize; return; } - if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) { - /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */ - dctxPtr->dictSize += dstSize; + if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) { + /* assumption : dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart */ + dctx->dictSize += dstSize; return; } if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ - size_t const preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - size_t copySize = 64 KB - dctxPtr->tmpOutSize; - const BYTE* const oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; - if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; + size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer; + size_t copySize = 64 KB - dctx->tmpOutSize; + const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; + if (dctx->tmpOutSize > 64 KB) copySize = 0; if (copySize > preserveSize) copySize = preserveSize; - memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize; return; } - if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { /* copy dst into tmp to complete dict */ - if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) { /* tmp buffer not large enough */ + if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */ + if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */ size_t const preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); - dctxPtr->dictSize = preserveSize; + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); + dctx->dictSize = preserveSize; } - memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize); - dctxPtr->dictSize += dstSize; + memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize); + dctx->dictSize += dstSize; return; } /* join dict & dest into tmp */ { size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ - if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize; - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); - memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dstSize; + if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize; + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); + memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize); + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dstSize; } } /*! LZ4F_decompress() : - * Call this function repetitively to regenerate data compressed within srcBuffer. - * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. + * Call this function repetitively to regenerate compressed data in srcBuffer. + * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer + * into dstBuffer of capacity *dstSizePtr. * - * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). * - * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). - * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. - * Remaining data will have to be presented again in a subsequent invocation. + * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * If number of bytes read is < number of bytes provided, then decompression operation is not complete. + * Remaining data will have to be presented again in a subsequent invocation. * - * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. - * Basically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some boost to performance, since it allows less buffer shuffling. - * Note that this is just a hint, it's always possible to any srcSize value. - * When a frame is fully decoded, @return will be 0. - * If decompression failed, @return is an error code which can be tested using LZ4F_isError(). + * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling. + * Note that this is just a hint, and it's always possible to any srcSize value. + * When a frame is fully decoded, @return will be 0. + * If decompression failed, @return is an error code which can be tested using LZ4F_isError(). */ -size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, +size_t LZ4F_decompress(LZ4F_dctx* dctx, void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, const LZ4F_decompressOptions_t* decompressOptionsPtr) @@ -1078,257 +1228,288 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, *srcSizePtr = 0; *dstSizePtr = 0; - /* behaves like a state machine */ + /* behaves as a state machine */ while (doAnotherStage) { - switch(dctxPtr->dStage) + switch(dctx->dStage) { - case dstage_getHeader: + case dstage_getFrameHeader: if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */ - LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); /* will change dStage appropriately */ + size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr); /* will update dStage appropriately */ if (LZ4F_isError(hSize)) return hSize; srcPtr += hSize; break; } - dctxPtr->tmpInSize = 0; + dctx->tmpInSize = 0; if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */ - dctxPtr->tmpInTarget = minFHSize; /* minimum to attempt decode */ - dctxPtr->dStage = dstage_storeHeader; + dctx->tmpInTarget = minFHSize; /* minimum to attempt decode */ + dctx->dStage = dstage_storeFrameHeader; /* fall-through */ - case dstage_storeHeader: - { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - dctxPtr->tmpInSize += sizeToCopy; + case dstage_storeFrameHeader: + { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr)); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { - nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ + if (dctx->tmpInSize < dctx->tmpInTarget) { + nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ doAnotherStage = 0; /* not enough src data, ask for some more */ break; } - { LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); /* will change dStage appropriately */ + { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */ if (LZ4F_isError(hSize)) return hSize; } break; } case dstage_init: - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); + if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0); /* internal buffers allocation */ - { size_t const bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB); - if (bufferNeeded > dctxPtr->maxBufferSize) { /* tmp buffers too small */ - dctxPtr->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ - FREEMEM(dctxPtr->tmpIn); - dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize); - if (dctxPtr->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed); - FREEMEM(dctxPtr->tmpOutBuffer); - dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded); - if (dctxPtr->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed); - dctxPtr->maxBufferSize = bufferNeeded; + { size_t const bufferNeeded = dctx->maxBlockSize + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB) + 4 /* block checksum */; + if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ + dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ + FREEMEM(dctx->tmpIn); + dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize); + if (dctx->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed); + FREEMEM(dctx->tmpOutBuffer); + dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded); + if (dctx->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed); + dctx->maxBufferSize = bufferNeeded; } } - dctxPtr->tmpInSize = 0; - dctxPtr->tmpInTarget = 0; - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = 0; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; - dctxPtr->tmpOutStart = 0; - dctxPtr->tmpOutSize = 0; - - dctxPtr->dStage = dstage_getCBlockSize; + dctx->tmpInSize = 0; + dctx->tmpInTarget = 0; + dctx->tmpOut = dctx->tmpOutBuffer; + dctx->tmpOutStart = 0; + dctx->tmpOutSize = 0; + + dctx->dStage = dstage_getBlockHeader; /* fall-through */ - case dstage_getCBlockSize: + case dstage_getBlockHeader: if ((size_t)(srcEnd - srcPtr) >= BHSize) { selectedIn = srcPtr; srcPtr += BHSize; } else { /* not enough input to read cBlockSize field */ - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlockSize; + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeBlockHeader; } - if (dctxPtr->dStage == dstage_storeCBlockSize) /* can be skipped */ - case dstage_storeCBlockSize: - { size_t sizeToCopy = BHSize - dctxPtr->tmpInSize; + if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */ + case dstage_storeBlockHeader: + { size_t sizeToCopy = BHSize - dctx->tmpInSize; if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < BHSize) { /* not enough input for cBlockSize */ - nextSrcSizeHint = BHSize - dctxPtr->tmpInSize; + dctx->tmpInSize += sizeToCopy; + if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */ + nextSrcSizeHint = BHSize - dctx->tmpInSize; doAnotherStage = 0; break; } - selectedIn = dctxPtr->tmpIn; + selectedIn = dctx->tmpIn; } - /* case dstage_decodeCBlockSize: */ /* no more direct access, to remove scan-build warning */ + /* decode block header */ { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; - if (nextCBlockSize==0) { /* frameEnd signal, no more CBlock */ - dctxPtr->dStage = dstage_getSuffix; + size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4; + if (nextCBlockSize==0) { /* frameEnd signal, no more block */ + dctx->dStage = dstage_getSuffix; break; } - if (nextCBlockSize > dctxPtr->maxBlockSize) + if (nextCBlockSize > dctx->maxBlockSize) return err0r(LZ4F_ERROR_maxBlockSize_invalid); - dctxPtr->tmpInTarget = nextCBlockSize; if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) { - dctxPtr->dStage = dstage_copyDirect; + /* next block is uncompressed */ + dctx->tmpInTarget = nextCBlockSize; + if (dctx->frameInfo.blockChecksumFlag) { + XXH32_reset(&dctx->blockChecksum, 0); + } + dctx->dStage = dstage_copyDirect; break; } - dctxPtr->dStage = dstage_getCBlock; + /* next block is a compressed block */ + dctx->tmpInTarget = nextCBlockSize + crcSize; + dctx->dStage = dstage_getCBlock; if (dstPtr==dstEnd) { - nextSrcSizeHint = nextCBlockSize + BHSize; + nextSrcSizeHint = nextCBlockSize + crcSize + BHSize; doAnotherStage = 0; } break; } case dstage_copyDirect: /* uncompressed block */ - { size_t sizeToCopy = dctxPtr->tmpInTarget; - if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; - if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; + { size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr)); + size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); memcpy(dstPtr, srcPtr, sizeToCopy); - if (dctxPtr->frameInfo.contentChecksumFlag) - XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy); - if (dctxPtr->frameInfo.contentSize) - dctxPtr->frameRemainingSize -= sizeToCopy; + if (dctx->frameInfo.blockChecksumFlag) { + XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + } + if (dctx->frameInfo.contentChecksumFlag) + XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= sizeToCopy; - /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); + /* history management (linked blocks only)*/ + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) + LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0); srcPtr += sizeToCopy; dstPtr += sizeToCopy; - if (sizeToCopy == dctxPtr->tmpInTarget) { /* all copied */ - dctxPtr->dStage = dstage_getCBlockSize; + if (sizeToCopy == dctx->tmpInTarget) { /* all done */ + if (dctx->frameInfo.blockChecksumFlag) { + dctx->tmpInSize = 0; + dctx->dStage = dstage_getBlockChecksum; + } else + dctx->dStage = dstage_getBlockHeader; /* new block */ break; } - dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ - nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize; + dctx->tmpInTarget -= sizeToCopy; /* need to copy more */ + nextSrcSizeHint = dctx->tmpInTarget + + + dctx->frameInfo.contentChecksumFlag * 4 /* block checksum */ + + BHSize /* next header size */; doAnotherStage = 0; break; } + /* check block checksum for recently transferred uncompressed block */ + case dstage_getBlockChecksum: + { const void* crcSrc; + if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) { + crcSrc = srcPtr; + srcPtr += 4; + } else { + size_t const stillToCopy = 4 - dctx->tmpInSize; + size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr)); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctx->tmpInSize < 4) { /* all input consumed */ + doAnotherStage = 0; + break; + } + crcSrc = dctx->header; + } + { U32 const readCRC = LZ4F_readLE32(crcSrc); + U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); + if (readCRC != calcCRC) + return err0r(LZ4F_ERROR_blockChecksum_invalid); + } + } + dctx->dStage = dstage_getBlockHeader; /* new block */ + break; + case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */ - if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) { - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlock; + if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) { + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeCBlock; break; } + /* input large enough to read full block directly */ selectedIn = srcPtr; - srcPtr += dctxPtr->tmpInTarget; - dctxPtr->dStage = dstage_decodeCBlock; + srcPtr += dctx->tmpInTarget; + dctx->dStage = dstage_decodeCBlock; break; case dstage_storeCBlock: - { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - dctxPtr->tmpInSize += sizeToCopy; + { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd-srcPtr)); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* need more input */ - nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; + if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */ + nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; doAnotherStage=0; break; } - selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_decodeCBlock; + selectedIn = dctx->tmpIn; + dctx->dStage = dstage_decodeCBlock; } /* fall-through */ + /* At this stage, input is large enough to decode a block */ case dstage_decodeCBlock: - if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */ - dctxPtr->dStage = dstage_decodeCBlock_intoTmp; + if (dctx->frameInfo.blockChecksumFlag) { + dctx->tmpInTarget -= 4; + { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget); + U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0); + if (readBlockCrc != calcBlockCrc) + return err0r(LZ4F_ERROR_blockChecksum_invalid); + } } + if ((size_t)(dstEnd-dstPtr) < dctx->maxBlockSize) /* not enough place into dst : decode into tmpOut */ + dctx->dStage = dstage_decodeCBlock_intoTmp; else - dctxPtr->dStage = dstage_decodeCBlock_intoDst; + dctx->dStage = dstage_decodeCBlock_intoDst; break; case dstage_decodeCBlock_intoDst: - { int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; - - if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) - decoder = LZ4_decompress_safe_usingDict; - else - decoder = LZ4F_decompress_safe; - - decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, - (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, - (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + { int const decodedSize = LZ4_decompress_safe_usingDict( + (const char*)selectedIn, (char*)dstPtr, + (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, + (const char*)dctx->dict, (int)dctx->dictSize); if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) - XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); - if (dctxPtr->frameInfo.contentSize) - dctxPtr->frameRemainingSize -= decodedSize; + if (dctx->frameInfo.contentChecksumFlag) + XXH32_update(&(dctx->xxh), dstPtr, decodedSize); + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= decodedSize; /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); + if (dctx->frameInfo.blockMode==LZ4F_blockLinked) + LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0); dstPtr += decodedSize; - dctxPtr->dStage = dstage_getCBlockSize; + dctx->dStage = dstage_getBlockHeader; break; } case dstage_decodeCBlock_intoTmp: /* not enough place into dst : decode into tmpOut */ - { int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; - - if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) - decoder = LZ4_decompress_safe_usingDict; - else - decoder = LZ4F_decompress_safe; - - /* ensure enough place for tmpOut */ - if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) { - if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { - if (dctxPtr->dictSize > 128 KB) { - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB); - dctxPtr->dictSize = 64 KB; - } - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize; - } else { /* dict not within tmp */ - size_t reservedDictSpace = dctxPtr->dictSize; - if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; + + /* ensure enough place for tmpOut */ + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) { + if (dctx->dict == dctx->tmpOutBuffer) { + if (dctx->dictSize > 128 KB) { + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB); + dctx->dictSize = 64 KB; } + dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize; + } else { /* dict not within tmp */ + size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB); + dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace; } + } - /* Decode */ - decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, - (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, - (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) - return err0r(LZ4F_ERROR_decompressionFailed); /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) - XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); - if (dctxPtr->frameInfo.contentSize) - dctxPtr->frameRemainingSize -= decodedSize; - dctxPtr->tmpOutSize = decodedSize; - dctxPtr->tmpOutStart = 0; - dctxPtr->dStage = dstage_flushOut; - break; + /* Decode block */ + { int const decodedSize = LZ4_decompress_safe_usingDict( + (const char*)selectedIn, (char*)dctx->tmpOut, + (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, + (const char*)dctx->dict, (int)dctx->dictSize); + if (decodedSize < 0) /* decompression failed */ + return err0r(LZ4F_ERROR_decompressionFailed); + if (dctx->frameInfo.contentChecksumFlag) + XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize); + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= decodedSize; + dctx->tmpOutSize = decodedSize; + dctx->tmpOutStart = 0; + dctx->dStage = dstage_flushOut; } + /* fall-through */ case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ - { size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart; - if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr; - memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); + { size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr)); + memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy); /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); + if (dctx->frameInfo.blockMode==LZ4F_blockLinked) + LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1); - dctxPtr->tmpOutStart += sizeToCopy; + dctx->tmpOutStart += sizeToCopy; dstPtr += sizeToCopy; - /* end of flush ? */ - if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) { - dctxPtr->dStage = dstage_getCBlockSize; + if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */ + dctx->dStage = dstage_getBlockHeader; /* get next block */ break; } nextSrcSizeHint = BHSize; @@ -1337,46 +1518,47 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, } case dstage_getSuffix: - { size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; - if (dctxPtr->frameRemainingSize) + { size_t const suffixSize = dctx->frameInfo.contentChecksumFlag * 4; + if (dctx->frameRemainingSize) return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */ if (suffixSize == 0) { /* frame completed */ nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; + LZ4F_resetDecompressionContext(dctx); doAnotherStage = 0; break; } if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */ - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeSuffix; + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeSuffix; } else { selectedIn = srcPtr; srcPtr += 4; } } - if (dctxPtr->dStage == dstage_storeSuffix) /* can be skipped */ + if (dctx->dStage == dstage_storeSuffix) /* can be skipped */ case dstage_storeSuffix: { - size_t sizeToCopy = 4 - dctxPtr->tmpInSize; + size_t sizeToCopy = 4 - dctx->tmpInSize; if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < 4) { /* not enough input to read complete suffix */ - nextSrcSizeHint = 4 - dctxPtr->tmpInSize; + dctx->tmpInSize += sizeToCopy; + if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */ + nextSrcSizeHint = 4 - dctx->tmpInSize; doAnotherStage=0; break; } - selectedIn = dctxPtr->tmpIn; + selectedIn = dctx->tmpIn; } - /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */ + /* case dstage_checkSuffix: */ /* no direct call, avoid scan-build warning */ { U32 const readCRC = LZ4F_readLE32(selectedIn); - U32 const resultCRC = XXH32_digest(&(dctxPtr->xxh)); - if (readCRC != resultCRC) return err0r(LZ4F_ERROR_contentChecksum_invalid); + U32 const resultCRC = XXH32_digest(&(dctx->xxh)); + if (readCRC != resultCRC) + return err0r(LZ4F_ERROR_contentChecksum_invalid); nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; + LZ4F_resetDecompressionContext(dctx); doAnotherStage = 0; break; } @@ -1387,78 +1569,77 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, srcPtr += 4; } else { /* not enough input to read cBlockSize field */ - dctxPtr->tmpInSize = 4; - dctxPtr->tmpInTarget = 8; - dctxPtr->dStage = dstage_storeSFrameSize; + dctx->tmpInSize = 4; + dctx->tmpInTarget = 8; + dctx->dStage = dstage_storeSFrameSize; } - if (dctxPtr->dStage == dstage_storeSFrameSize) + if (dctx->dStage == dstage_storeSFrameSize) case dstage_storeSFrameSize: { - size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, + (size_t)(srcEnd - srcPtr) ); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { + dctx->tmpInSize += sizeToCopy; + if (dctx->tmpInSize < dctx->tmpInTarget) { /* not enough input to get full sBlockSize; wait for more */ - nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize; doAnotherStage = 0; break; } - selectedIn = dctxPtr->header + 4; + selectedIn = dctx->header + 4; } /* case dstage_decodeSFrameSize: */ /* no direct access */ { size_t const SFrameSize = LZ4F_readLE32(selectedIn); - dctxPtr->frameInfo.contentSize = SFrameSize; - dctxPtr->tmpInTarget = SFrameSize; - dctxPtr->dStage = dstage_skipSkippable; + dctx->frameInfo.contentSize = SFrameSize; + dctx->tmpInTarget = SFrameSize; + dctx->dStage = dstage_skipSkippable; break; } case dstage_skipSkippable: - { size_t skipSize = dctxPtr->tmpInTarget; - if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr; + { size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr)); srcPtr += skipSize; - dctxPtr->tmpInTarget -= skipSize; + dctx->tmpInTarget -= skipSize; doAnotherStage = 0; - nextSrcSizeHint = dctxPtr->tmpInTarget; - if (nextSrcSizeHint) break; - dctxPtr->dStage = dstage_getHeader; + nextSrcSizeHint = dctx->tmpInTarget; + if (nextSrcSizeHint) break; /* still more to skip */ + LZ4F_resetDecompressionContext(dctx); break; } } } - /* preserve dictionary within tmp if necessary */ - if ( (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) - &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) - &&(!decompressOptionsPtr->stableDst) - &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) - ) + /* preserve history within tmp if necessary */ + if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) + && (dctx->dict != dctx->tmpOutBuffer) + && (dctx->dStage != dstage_getFrameHeader) + && (!decompressOptionsPtr->stableDst) + && ((unsigned)(dctx->dStage-1) < (unsigned)(dstage_getSuffix-1)) ) { - if (dctxPtr->dStage == dstage_flushOut) { - size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - size_t copySize = 64 KB - dctxPtr->tmpOutSize; - const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; - if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; + if (dctx->dStage == dstage_flushOut) { + size_t preserveSize = dctx->tmpOut - dctx->tmpOutBuffer; + size_t copySize = 64 KB - dctx->tmpOutSize; + const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; + if (dctx->tmpOutSize > 64 KB) copySize = 0; if (copySize > preserveSize) copySize = preserveSize; - memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart; + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dctx->tmpOutStart; } else { - size_t newDictSize = dctxPtr->dictSize; - const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; + size_t newDictSize = dctx->dictSize; + const BYTE* oldDictEnd = dctx->dict + dctx->dictSize; if ((newDictSize) > 64 KB) newDictSize = 64 KB; - memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); + memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = newDictSize; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize; + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = newDictSize; + dctx->tmpOut = dctx->tmpOutBuffer + newDictSize; } } @@ -1466,3 +1647,23 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, *dstSizePtr = (dstPtr - dstStart); return nextSrcSizeHint; } + +/*! LZ4F_decompress_usingDict() : + * Same as LZ4F_decompress(), using a predefined dictionary. + * Dictionary is used "in place", without any preprocessing. + * It must remain accessible throughout the entire frame decoding. + */ +size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr) +{ + if (dctx->dStage <= dstage_init) { + dctx->dict = (const BYTE*)dict; + dctx->dictSize = dictSize; + } + return LZ4F_decompress(dctx, dstBuffer, dstSizePtr, + srcBuffer, srcSizePtr, + decompressOptionsPtr); +} diff --git a/lib/lz4frame.h b/lib/lz4frame.h index dd2be58..88a6513 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -100,8 +100,8 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return /*-************************************ * Frame compression types **************************************/ -/* #define LZ4F_DISABLE_OBSOLETE_ENUMS */ /* uncomment to disable obsolete enums */ -#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS +/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */ +#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS # define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x #else # define LZ4F_OBSOLETE_ENUM(x) @@ -140,12 +140,17 @@ typedef enum { } LZ4F_contentChecksum_t; typedef enum { + LZ4F_noBlockChecksum=0, + LZ4F_blockChecksumEnabled +} LZ4F_blockChecksum_t; + +typedef enum { LZ4F_frame=0, LZ4F_skippableFrame LZ4F_OBSOLETE_ENUM(skippableFrame) } LZ4F_frameType_t; -#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS +#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS typedef LZ4F_blockSizeID_t blockSizeID_t; typedef LZ4F_blockMode_t blockMode_t; typedef LZ4F_frameType_t frameType_t; @@ -153,50 +158,52 @@ typedef LZ4F_contentChecksum_t contentChecksum_t; #endif /*! LZ4F_frameInfo_t : - * makes it possible to supply detailed frame parameters to the stream interface. - * It's not required to set all fields, as long as the structure was initially memset() to zero. - * All reserved fields must be set to zero. */ + * makes it possible to set or read frame parameters. + * It's not required to set all fields, as long as the structure was initially memset() to zero. + * For all fields, 0 sets it to default value */ typedef struct { - LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ - LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ - LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */ - unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */ - unsigned reserved[2]; /* must be zero for forward compatibility */ + LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */ + LZ4F_contentChecksum_t contentChecksumFlag; /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */ + LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */ + unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */ + unsigned dictID; /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */ + LZ4F_blockChecksum_t blockChecksumFlag; /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */ } LZ4F_frameInfo_t; /*! LZ4F_preferences_t : - * makes it possible to supply detailed compression parameters to the stream interface. - * It's not required to set all fields, as long as the structure was initially memset() to zero. - * All reserved fields must be set to zero. */ + * makes it possible to supply detailed compression parameters to the stream interface. + * It's not required to set all fields, as long as the structure was initially memset() to zero. + * All reserved fields must be set to zero. */ typedef struct { LZ4F_frameInfo_t frameInfo; int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */ - unsigned autoFlush; /* 1 == always flush (reduce usage of tmp buffer) */ + unsigned autoFlush; /* 1 == always flush, to reduce usage of internal buffers */ unsigned reserved[4]; /* must be zero for forward compatibility */ } LZ4F_preferences_t; +LZ4FLIB_API int LZ4F_compressionLevel_max(void); + /*-********************************* * Simple compression function ***********************************/ -/*!LZ4F_compressFrameBound() : - * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. - * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. +/*! LZ4F_compressFrameBound() : + * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. + * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. */ LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); -/*!LZ4F_compressFrame() : - * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1 - * An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation. - * This value is supplied by LZ4F_compressFrameBound(). - * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode). - * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. +/*! LZ4F_compressFrame() : + * Compress an entire srcBuffer into a valid LZ4 frame. + * dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. * @return : number of bytes written into dstBuffer. * or an error code if it fails (can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr); - +LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr); /*-*********************************** @@ -228,15 +235,17 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /*---- Compression ----*/ -#define LZ4F_HEADER_SIZE_MAX 15 +#define LZ4F_HEADER_SIZE_MAX 19 /*! LZ4F_compressBegin() : - * will write the frame header into dstBuffer. - * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. + * will write the frame header into dstBuffer. + * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. * @return : number of bytes written into dstBuffer for the header * or an error code (which can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr); +LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* prefsPtr); /*! LZ4F_compressBound() : * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. @@ -269,13 +278,13 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); /*! LZ4F_compressEnd() : - * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). - * It will flush whatever data remained within `cctx` (like LZ4_flush()) - * and properly finalize the frame, with an endMark and a checksum. + * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). + * It will flush whatever data remained within `cctx` (like LZ4_flush()) + * and properly finalize the frame, with an endMark and a checksum. * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) * or an error code if it fails (which can be tested using LZ4F_isError()) - * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. + * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. */ LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); @@ -288,7 +297,7 @@ typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous typedef struct { unsigned stableDst; /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */ - unsigned reserved[3]; + unsigned reserved[3]; /* must be set to zero for forward compatibility */ } LZ4F_decompressOptions_t; @@ -312,10 +321,10 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); *************************************/ /*! LZ4F_getFrameInfo() : - * This function extracts frame parameters (such as max blockSize, frame checksum, etc.). - * Its usage is optional. - * Extracted information can typically be useful for allocation purposes. - * This function works in 2 situations : + * This function extracts frame parameters (max blockSize, dictID, etc.). + * Its usage is optional. + * Extracted information is typically useful for allocation and dictionary. + * This function works in 2 situations : * - At the beginning of a new frame, in which case * it will decode information from `srcBuffer`, starting the decoding process. * Input size must be large enough to successfully decode the entire frame header. @@ -323,41 +332,42 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); * It's allowed to provide more input data than this minimum. * - After decoding has been started. * In which case, no input is read, frame parameters are extracted from dctx. - * - If decoding has barely started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail. - * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - * Decompression must resume from (srcBuffer + *srcSizePtr). + * - If decoding has barely started, but not yet extracted information from header, + * LZ4F_getFrameInfo() will fail. + * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + * Decompression must resume from (srcBuffer + *srcSizePtr). * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, - * or an error code which can be tested using LZ4F_isError() - * note 1 : in case of error, dctx is not modified. Decoding operation can resume safely. - * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. + * or an error code which can be tested using LZ4F_isError(). + * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. + * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. */ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr); /*! LZ4F_decompress() : - * Call this function repetitively to regenerate compressed data from `srcBuffer`. - * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. + * Call this function repetitively to regenerate compressed data from `srcBuffer`. + * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. * - * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value). + * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value). * - * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value). - * Number of bytes consumed can be < number of bytes provided. - * It typically happens when dstBuffer is not large enough to contain all decoded data. - * Unconsumed source data must be presented again in subsequent invocations. + * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value). + * Number of bytes consumed can be < number of bytes provided. + * It typically happens when dstBuffer is not large enough to contain all decoded data. + * Unconsumed source data must be presented again in subsequent invocations. * * `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten. * `dstBuffer` itself can be changed at will between each consecutive function invocation. * - * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. - * Schematically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some small speed benefit, because it skips intermediate buffers. - * This is just a hint though, it's always possible to provide any srcSize. - * When a frame is fully decoded, @return will be 0 (no more data expected). - * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). + * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides some small speed benefit, because it skips intermediate buffers. + * This is just a hint though, it's always possible to provide any srcSize. + * When a frame is fully decoded, @return will be 0 (no more data expected). + * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). * - * After a frame is fully decoded, dctx can be used again to decompress another frame. - * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state. + * After a frame is fully decoded, dctx can be used again to decompress another frame. + * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state. */ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx, void* dstBuffer, size_t* dstSizePtr, diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index d3bae82..1899f8e 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -59,7 +59,7 @@ extern "C" { ITEM(ERROR_contentChecksumFlag_invalid) \ ITEM(ERROR_compressionLevel_invalid) \ ITEM(ERROR_headerVersion_wrong) \ - ITEM(ERROR_blockChecksum_unsupported) \ + ITEM(ERROR_blockChecksum_invalid) \ ITEM(ERROR_reservedFlag_set) \ ITEM(ERROR_allocation_failed) \ ITEM(ERROR_srcSize_tooLarge) \ @@ -82,6 +82,60 @@ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); + +/********************************** + * Bulk processing dictionary API + *********************************/ +typedef struct LZ4F_CDict_s LZ4F_CDict; + +/*! LZ4_createCDict() : + * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */ +LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); +void LZ4F_freeCDict(LZ4F_CDict* CDict); + + +/*! LZ4_compressFrame_usingCDict() : + * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary. + * If cdict==NULL, compress without a dictionary. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * If this condition is not respected, function will fail (@return an errorCode). + * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, + * but it's not recommended, as it's the only way to provide dictID in the frame header. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) */ +size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr); + + +/*! LZ4F_compressBegin_usingCDict() : + * Inits streaming dictionary compression, and writes the frame header into dstBuffer. + * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * `prefsPtr` is optional : you may provide NULL as argument, + * however, it's the only way to provide dictID in the frame header. + * @return : number of bytes written into dstBuffer for the header, + * or an error code (which can be tested using LZ4F_isError()) */ +size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* prefsPtr); + + +/*! LZ4F_decompress_usingDict() : + * Same as LZ4F_decompress(), using a predefined dictionary. + * Dictionary is used "in place", without any preprocessing. + * It must remain accessible throughout the entire frame decoding. */ +size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr); + + #if defined (__cplusplus) } #endif diff --git a/lib/lz4hc.c b/lib/lz4hc.c index ca9c2e6..22eb071 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -69,6 +69,8 @@ /*=== Macros ===*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) #define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ @@ -610,7 +612,11 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i **************************************/ /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } -int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; } +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { + if (!LZ4_streamHCPtr) return 0; /* support free on NULL */ + free(LZ4_streamHCPtr); + return 0; +} /* initialization */ @@ -623,6 +629,16 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel); } +void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + int const currentCLevel = LZ4_streamHCPtr->internal_donotuse.compressionLevel; + int const minCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? 1 : LZ4HC_CLEVEL_OPT_MIN; + int const maxCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? LZ4HC_CLEVEL_OPT_MIN-1 : LZ4HC_CLEVEL_MAX; + compressionLevel = MIN(compressionLevel, minCLevel); + compressionLevel = MAX(compressionLevel, maxCLevel); + LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel; +} + int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; @@ -767,7 +783,11 @@ void* LZ4_createHC (char* inputBuffer) return hc4; } -int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; } +int LZ4_freeHC (void* LZ4HC_Data) { + if (!LZ4HC_Data) return 0; /* support free on NULL */ + FREEMEM(LZ4HC_Data); + return 0; +} int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel) { diff --git a/lib/lz4hc.h b/lib/lz4hc.h index b63c825..66d5636 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -161,7 +161,7 @@ typedef struct typedef struct { unsigned int hashTable[LZ4HC_HASHTABLESIZE]; - unsigned short chainTable[LZ4HC_MAXD]; + unsigned short chainTable[LZ4HC_MAXD]; const unsigned char* end; /* next block here to continue on current prefix */ const unsigned char* base; /* All index relative to this position */ const unsigned char* dictBase; /* alternate base for extDict */ @@ -245,25 +245,34 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStr * or 0 if compression fails. * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src` */ -LZ4LIB_API int LZ4_compress_HC_destSize(void* LZ4HC_Data, +int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int compressionLevel); -/*! LZ4_compress_HC_continue_destSize() : +/*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental) * Similar as LZ4_compress_HC_continue(), * but will read a variable nb of bytes from `src` * to fit into `targetDstSize` budget. * Result is provided in 2 parts : * @return : the number of bytes written into 'dst' * or 0 if compression fails. - * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src` - * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN - * beyond that level, compression performance will be much reduced due to internal incompatibilities + * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`. + * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN + * beyond that level, compression performance will be much reduced due to internal incompatibilities */ -LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, +int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDstSize); +/*! LZ4_setCompressionLevel() : v1.8.0 (experimental) + * It's possible to change compression level after LZ4_resetStreamHC(), between 2 invocations of LZ4_compress_HC_continue*(), + * but that requires to stay in the same mode (aka 1-10 or 11-12). + * This function ensures this condition. + */ +void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); + + + #endif /* LZ4_HC_SLO_098092834 */ #endif /* LZ4_HC_STATIC_LINKING_ONLY */ |