summaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
authorYann Collet <cyan@fb.com>2017-08-10 19:12:53 (GMT)
committerYann Collet <cyan@fb.com>2017-08-10 19:12:53 (GMT)
commit4531637ecdc4b12154c66cc69cfaade185039e4c (patch)
tree4d05cbe4542f63c111e35a099f83f537ef6a93d9 /lib
parent8d597d62d5455f5d315782048b9d32f129a60bac (diff)
downloadlz4-4531637ecdc4b12154c66cc69cfaade185039e4c.zip
lz4-4531637ecdc4b12154c66cc69cfaade185039e4c.tar.gz
lz4-4531637ecdc4b12154c66cc69cfaade185039e4c.tar.bz2
support dictionary compression with independent blocks
Diffstat (limited to 'lib')
-rw-r--r--lib/lz4frame.c82
-rw-r--r--lib/lz4frame.h63
2 files changed, 83 insertions, 62 deletions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 9aeb456..fbccc9d 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -171,7 +171,7 @@ typedef struct LZ4F_cctx_s
LZ4F_preferences_t prefs;
U32 version;
U32 cStage;
- int dictionary;
+ const LZ4F_CDict* cdict;
size_t maxBlockSize;
size_t maxBufferSize;
BYTE* tmpBuff;
@@ -534,19 +534,22 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
XXH32_reset(&(cctxPtr->xxh), 0);
/* context init */
- cctxPtr->dictionary = (cdict!=NULL);
- if (cdict) {
- if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
- memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+ cctxPtr->cdict = cdict;
+ if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
+ /* frame init only for blockLinked : blockIndependent will be init at each block */
+ if (cdict) {
+ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+ memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+ } else {
+ memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
+ LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ }
} else {
- memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
- LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+ LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
+ else
+ LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
}
- } else {
- if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
- LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
- else
- LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
}
/* Magic Number */
@@ -609,13 +612,15 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP
}
-typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level);
+typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict);
-static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level)
+static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level, const LZ4F_CDict* cdict)
{
/* compress a single block */
BYTE* const cSizePtr = (BYTE*)dst;
- U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level);
+ U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4),
+ (int)(srcSize), (int)(srcSize-1),
+ level, cdict);
LZ4F_writeLE32(cSizePtr, cSize);
if (cSize == 0) { /* compression failed */
cSize = (U32)srcSize;
@@ -626,32 +631,47 @@ static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, com
}
-static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
int const acceleration = (level < -1) ? -level : 1;
+ if (cdict) {
+ memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
+ return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
+ }
return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration);
}
-static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
int const acceleration = (level < -1) ? -level : 1;
+ (void)cdict; /* init once at beginning of frame */
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
}
-static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
+{
+ if (cdict) {
+ memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
+ LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
+ return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
+ }
+ return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
+}
+
+static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
- (void) level;
+ (void)level; (void)cdict; /* init once at beginning of frame */
return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
}
-static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int dictionary, int level)
+static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
{
if (level < LZ4HC_CLEVEL_MIN) {
- if (blockMode == LZ4F_blockIndependent && !dictionary) return LZ4F_localLZ4_compress_limitedOutput_withState;
- return LZ4F_localLZ4_compress_limitedOutput_continue;
+ if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
+ return LZ4F_compressBlock_continue;
}
- if (blockMode == LZ4F_blockIndependent && !dictionary) return LZ4_compress_HC_extStateHC;
- return LZ4F_localLZ4_compressHC_limitedOutput_continue;
+ if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC;
+ return LZ4F_compressBlockHC_continue;
}
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
@@ -679,7 +699,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
LZ4F_lastBlockStatus lastBlockCompressed = notDone;
- compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->dictionary, cctxPtr->prefs.compressionLevel);
+ compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
@@ -702,7 +722,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
srcPtr += sizeToCopy;
- dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
cctxPtr->tmpInSize = 0;
@@ -710,16 +730,16 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
}
while ((size_t)(srcEnd - srcPtr) >= blockSize) {
- /* compress full block */
+ /* compress full blocks */
lastBlockCompressed = fromSrcBuffer;
- dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
srcPtr += blockSize;
}
if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
/* compress remaining input < blockSize */
lastBlockCompressed = fromSrcBuffer;
- dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
srcPtr = srcEnd;
}
@@ -778,10 +798,10 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
(void)compressOptionsPtr; /* not yet useful */
/* select compression function */
- compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->dictionary, cctxPtr->prefs.compressionLevel);
+ compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
/* compress tmp buffer */
- dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
cctxPtr->tmpInSize = 0;
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index ccacb89..dd76194 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -100,8 +100,8 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return
/*-************************************
* Frame compression types
**************************************/
-/* #define LZ4F_DISABLE_OBSOLETE_ENUMS */ /* uncomment to disable obsolete enums */
-#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */
+#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
# define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x
#else
# define LZ4F_OBSOLETE_ENUM(x)
@@ -145,7 +145,7 @@ typedef enum {
LZ4F_OBSOLETE_ENUM(skippableFrame)
} LZ4F_frameType_t;
-#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
typedef LZ4F_blockSizeID_t blockSizeID_t;
typedef LZ4F_blockMode_t blockMode_t;
typedef LZ4F_frameType_t frameType_t;
@@ -183,9 +183,9 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void);
/*-*********************************
* Simple compression function
***********************************/
-/*!LZ4F_compressFrameBound() :
- * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
- * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+/*! LZ4F_compressFrameBound() :
+ * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
+ * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
*/
LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
@@ -316,10 +316,10 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
*************************************/
/*! LZ4F_getFrameInfo() :
- * This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- * Its usage is optional.
- * Extracted information can typically be useful for allocation purposes.
- * This function works in 2 situations :
+ * This function extracts frame parameters (max blockSize, dictID, etc.).
+ * Its usage is optional.
+ * Extracted information is typically useful for allocation and dictionary.
+ * This function works in 2 situations :
* - At the beginning of a new frame, in which case
* it will decode information from `srcBuffer`, starting the decoding process.
* Input size must be large enough to successfully decode the entire frame header.
@@ -327,41 +327,42 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
* It's allowed to provide more input data than this minimum.
* - After decoding has been started.
* In which case, no input is read, frame parameters are extracted from dctx.
- * - If decoding has barely started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
- * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- * Decompression must resume from (srcBuffer + *srcSizePtr).
+ * - If decoding has barely started, but not yet extracted information from header,
+ * LZ4F_getFrameInfo() will fail.
+ * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ * Decompression must resume from (srcBuffer + *srcSizePtr).
* @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
- * or an error code which can be tested using LZ4F_isError()
- * note 1 : in case of error, dctx is not modified. Decoding operation can resume safely.
- * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
+ * or an error code which can be tested using LZ4F_isError().
+ * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
+ * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
*/
LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
/*! LZ4F_decompress() :
- * Call this function repetitively to regenerate compressed data from `srcBuffer`.
- * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+ * Call this function repetitively to regenerate compressed data from `srcBuffer`.
+ * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
*
- * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
+ * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
*
- * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
- * Number of bytes consumed can be < number of bytes provided.
- * It typically happens when dstBuffer is not large enough to contain all decoded data.
- * Unconsumed source data must be presented again in subsequent invocations.
+ * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
+ * Number of bytes consumed can be < number of bytes provided.
+ * It typically happens when dstBuffer is not large enough to contain all decoded data.
+ * Unconsumed source data must be presented again in subsequent invocations.
*
* `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
* `dstBuffer` itself can be changed at will between each consecutive function invocation.
*
- * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
- * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
- * Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
- * This is just a hint though, it's always possible to provide any srcSize.
- * When a frame is fully decoded, @return will be 0 (no more data expected).
- * If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
+ * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
+ * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ * Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
+ * This is just a hint though, it's always possible to provide any srcSize.
+ * When a frame is fully decoded, @return will be 0 (no more data expected).
+ * If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
*
- * After a frame is fully decoded, dctx can be used again to decompress another frame.
- * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
+ * After a frame is fully decoded, dctx can be used again to decompress another frame.
+ * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
*/
LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
void* dstBuffer, size_t* dstSizePtr,