support dictionary compression with independent blocks

2 files changed, 83 insertions, 62 deletions

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 9aeb456..fbccc9d 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -171,7 +171,7 @@ typedef struct LZ4F_cctx_s
     LZ4F_preferences_t prefs;
     U32    version;
     U32    cStage;
-    int    dictionary;
+    const LZ4F_CDict* cdict;
     size_t maxBlockSize;
     size_t maxBufferSize;
     BYTE*  tmpBuff;
@@ -534,19 +534,22 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     XXH32_reset(&(cctxPtr->xxh), 0);
 
     /* context init */
-    cctxPtr->dictionary = (cdict!=NULL);
-    if (cdict) {
-        if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-            memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+    cctxPtr->cdict = cdict;
+    if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
+        /* frame init only for blockLinked : blockIndependent will be init at each block */
+        if (cdict) {
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+                memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+            } else {
+                memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
+                LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+            }
         } else {
-            memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
-            LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+                LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
+            else
+                LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
         }
-    } else {
-        if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-            LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
-        else
-            LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
     }
 
     /* Magic Number */
@@ -609,13 +612,15 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP
 }
 
 
-typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level);
+typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict);
 
-static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level)
+static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level, const LZ4F_CDict* cdict)
 {
     /* compress a single block */
     BYTE* const cSizePtr = (BYTE*)dst;
-    U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level);
+    U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4),
+                                      (int)(srcSize), (int)(srcSize-1),
+                                      level, cdict);
     LZ4F_writeLE32(cSizePtr, cSize);
     if (cSize == 0) {  /* compression failed */
         cSize = (U32)srcSize;
@@ -626,32 +631,47 @@ static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, com
 }
 
 
-static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
+    if (cdict) {
+        memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
+        return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
+    }
     return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
-static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
+    (void)cdict; /* init once at beginning of frame */
     return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
-static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
+{
+    if (cdict) {
+        memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
+        LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
+        return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
+    }
+    return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
+}
+
+static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
-    (void) level;
+    (void)level; (void)cdict; /* init once at beginning of frame */
     return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
 }
 
-static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int dictionary, int level)
+static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
 {
     if (level < LZ4HC_CLEVEL_MIN) {
-        if (blockMode == LZ4F_blockIndependent && !dictionary) return LZ4F_localLZ4_compress_limitedOutput_withState;
-        return LZ4F_localLZ4_compress_limitedOutput_continue;
+        if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
+        return LZ4F_compressBlock_continue;
     }
-    if (blockMode == LZ4F_blockIndependent && !dictionary) return LZ4_compress_HC_extStateHC;
-    return LZ4F_localLZ4_compressHC_limitedOutput_continue;
+    if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC;
+    return LZ4F_compressBlockHC_continue;
 }
 
 static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
@@ -679,7 +699,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
     BYTE* const dstStart = (BYTE*)dstBuffer;
     BYTE* dstPtr = dstStart;
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
-    compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->dictionary, cctxPtr->prefs.compressionLevel);
+    compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
@@ -702,7 +722,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
             memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
             srcPtr += sizeToCopy;
 
-            dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+            dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
 
             if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
             cctxPtr->tmpInSize = 0;
@@ -710,16 +730,16 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
     }
 
     while ((size_t)(srcEnd - srcPtr) >= blockSize) {
-        /* compress full block */
+        /* compress full blocks */
         lastBlockCompressed = fromSrcBuffer;
-        dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+        dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
         srcPtr += blockSize;
     }
 
     if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
         /* compress remaining input < blockSize */
         lastBlockCompressed = fromSrcBuffer;
-        dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+        dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
         srcPtr  = srcEnd;
     }
 
@@ -778,10 +798,10 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
     (void)compressOptionsPtr;   /* not yet useful */
 
     /* select compression function */
-    compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->dictionary, cctxPtr->prefs.compressionLevel);
+    compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
     /* compress tmp buffer */
-    dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+    dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
     if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
     cctxPtr->tmpInSize = 0;
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index ccacb89..dd76194 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -100,8 +100,8 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return
 /*-************************************
  *  Frame compression types
  **************************************/
-/* #define LZ4F_DISABLE_OBSOLETE_ENUMS */  /* uncomment to disable obsolete enums */
-#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+/* #define LZ4F_ENABLE_OBSOLETE_ENUMS   // uncomment to enable obsolete enums */
+#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
 #  define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x
 #else
 #  define LZ4F_OBSOLETE_ENUM(x)
@@ -145,7 +145,7 @@ typedef enum {
     LZ4F_OBSOLETE_ENUM(skippableFrame)
 } LZ4F_frameType_t;
 
-#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
 typedef LZ4F_blockSizeID_t blockSizeID_t;
 typedef LZ4F_blockMode_t blockMode_t;
 typedef LZ4F_frameType_t frameType_t;
@@ -183,9 +183,9 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void);
 /*-*********************************
 *  Simple compression function
 ***********************************/
-/*!LZ4F_compressFrameBound() :
- * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
- * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+/*! LZ4F_compressFrameBound() :
+ *  Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
+ *  Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
  */
 LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
 
@@ -316,10 +316,10 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
 *************************************/
 
 /*! LZ4F_getFrameInfo() :
- * This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- * Its usage is optional.
- * Extracted information can typically be useful for allocation purposes.
- * This function works in 2 situations :
+ *  This function extracts frame parameters (max blockSize, dictID, etc.).
+ *  Its usage is optional.
+ *  Extracted information is typically useful for allocation and dictionary.
+ *  This function works in 2 situations :
  *   - At the beginning of a new frame, in which case
  *     it will decode information from `srcBuffer`, starting the decoding process.
  *     Input size must be large enough to successfully decode the entire frame header.
@@ -327,41 +327,42 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
  *     It's allowed to provide more input data than this minimum.
  *   - After decoding has been started.
  *     In which case, no input is read, frame parameters are extracted from dctx.
- *   - If decoding has barely started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
- * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- * Decompression must resume from (srcBuffer + *srcSizePtr).
+ *   - If decoding has barely started, but not yet extracted information from header,
+ *     LZ4F_getFrameInfo() will fail.
+ *  The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ *  Decompression must resume from (srcBuffer + *srcSizePtr).
  * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
- *           or an error code which can be tested using LZ4F_isError()
- * note 1 : in case of error, dctx is not modified. Decoding operation can resume safely.
- * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
+ *           or an error code which can be tested using LZ4F_isError().
+ *  note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
+ *  note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
  */
 LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
                                      LZ4F_frameInfo_t* frameInfoPtr,
                                      const void* srcBuffer, size_t* srcSizePtr);
 
 /*! LZ4F_decompress() :
- * Call this function repetitively to regenerate compressed data from `srcBuffer`.
- * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+ *  Call this function repetitively to regenerate compressed data from `srcBuffer`.
+ *  The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
  *
- * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
+ *  The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
  *
- * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
- * Number of bytes consumed can be < number of bytes provided.
- * It typically happens when dstBuffer is not large enough to contain all decoded data.
- * Unconsumed source data must be presented again in subsequent invocations.
+ *  The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
+ *  Number of bytes consumed can be < number of bytes provided.
+ *  It typically happens when dstBuffer is not large enough to contain all decoded data.
+ *  Unconsumed source data must be presented again in subsequent invocations.
  *
  * `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
  * `dstBuffer` itself can be changed at will between each consecutive function invocation.
  *
- * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
- * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
- * Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
- * This is just a hint though, it's always possible to provide any srcSize.
- * When a frame is fully decoded, @return will be 0 (no more data expected).
- * If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
+ * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
+ *  Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ *  Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
+ *  This is just a hint though, it's always possible to provide any srcSize.
+ *  When a frame is fully decoded, @return will be 0 (no more data expected).
+ *  If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
  *
- * After a frame is fully decoded, dctx can be used again to decompress another frame.
- * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
+ *  After a frame is fully decoded, dctx can be used again to decompress another frame.
+ *  After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
  */
 LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
                                    void* dstBuffer, size_t* dstSizePtr,