lz4frame : implemented option stableSrc

Improved LZ4_compressFrame() speed

3 files changed, 28 insertions, 21 deletions

diff --git a/lz4frame.c b/lz4frame.c
index 4c63227..7290972 100644
--- a/lz4frame.c
+++ b/lz4frame.c
@@ -232,8 +232,9 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
  */
 size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
 {
-	LZ4F_preferences_t prefs = { 0 };
-	LZ4F_compressionContext_t cctx = NULL;
+    LZ4F_cctx_internal_t cctxI = { 0 };   /* works because no allocation */
+    LZ4F_preferences_t prefs = { 0 };
+	LZ4F_compressOptions_t options = { 0 };
 	LZ4F_errorCode_t errorCode;
 	BYTE* const dstStart = (BYTE*) dstBuffer;
 	BYTE* dstPtr = dstStart;
@@ -241,30 +242,27 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
 
 
     if (preferencesPtr!=NULL) prefs = *preferencesPtr;
+    cctxI.version = LZ4F_VERSION;
+    cctxI.maxBufferSize = 64 KB;   /* mess with real buffer size, to prevent allocation; works because autoflush==1 & stableSrc==1 */
     prefs.autoFlush = 1;
+    options.stableSrc = 1;
 
 	if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs))
 		return -ERROR_dstMaxSize_tooSmall;
 
-	errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
-	if (LZ4F_isError(errorCode)) return errorCode;
-
-	errorCode = LZ4F_compressBegin(cctx, dstBuffer, dstMaxSize, &prefs);  /* write header */
+	errorCode = LZ4F_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs);  /* write header */
 	if (LZ4F_isError(errorCode)) return errorCode;
 	dstPtr += errorCode;   /* header size */
 
 	dstMaxSize -= errorCode;
-    errorCode = LZ4F_compressUpdate(cctx, dstPtr, dstMaxSize, srcBuffer, srcSize, NULL);
+    errorCode = LZ4F_compressUpdate(&cctxI, dstPtr, dstMaxSize, srcBuffer, srcSize, &options);
 	if (LZ4F_isError(errorCode)) return errorCode;
 	dstPtr += errorCode;
 
-	errorCode = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, NULL);   /* flush last block, and generate suffix */
+	errorCode = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options);   /* flush last block, and generate suffix */
 	if (LZ4F_isError(errorCode)) return errorCode;
 	dstPtr += errorCode;
 
-	errorCode = LZ4F_freeCompressionContext(cctx);
-	if (LZ4F_isError(errorCode)) return errorCode;
-
 	return (dstPtr - dstStart);
 }
 
@@ -496,9 +494,9 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
         srcPtr += iSize;
     }
 
-	if ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) && (lastBlockCompressed))
+    /* save last input up to 64 KB for dictionary */
+	if ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) && (lastBlockCompressed) && (!compressOptionsPtr->stableSrc))
     {
-        /* last compressed input up to 64 KB become dictionary */
         if ((lastBlockCompressed==2) ||
            ((cctxPtr->tmpBuff + cctxPtr->maxBufferSize) < (cctxPtr->tmpIn + cctxPtr->maxBlockSize)))
         {
diff --git a/lz4frame.h b/lz4frame.h
index 25cf043..d75d462 100644
--- a/lz4frame.h
+++ b/lz4frame.h
@@ -85,7 +85,7 @@ typedef struct {
   blockSizeID_t     blockSizeID;           /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
   blockMode_t       blockMode;             /* blockLinked, blockIndependent ; 0 == default */
   contentChecksum_t contentChecksumFlag;   /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
-  unsigned          reserved[3];
+  unsigned          reserved[5];
 } LZ4F_frameInfo_t;
 
 typedef struct {
@@ -122,8 +122,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
 typedef void* LZ4F_compressionContext_t;
 
 typedef struct {
-  unsigned stableSrc;          /* unused for the time being, must be 0 */
-  unsigned reserved[5];
+  unsigned stableSrc;    /* 1 == src content will remain available on future calls to LZ4F_compress(); avoid saving src content within tmp buffer as future dictionary */
+  unsigned reserved[3];
 } LZ4F_compressOptions_t;
 
 /* Resource Management */
@@ -199,7 +199,7 @@ typedef void* LZ4F_decompressionContext_t;
 
 typedef struct {
   unsigned stableDst;       /* unused for the time being, must be 0 */
-  unsigned reserved[5];
+  unsigned reserved[3];
 } LZ4F_decompressOptions_t;
 
 /* Resource management */
diff --git a/programs/frametest.c b/programs/frametest.c
index c3662b4..73d6373 100644
--- a/programs/frametest.c
+++ b/programs/frametest.c
@@ -405,13 +405,19 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
         unsigned BMId   = FUZ_rand(&randState) & 1;
         unsigned CCflag = FUZ_rand(&randState) & 1;
         unsigned autoflush = (FUZ_rand(&randState) & 3) == 2;
-        LZ4F_preferences_t prefs = { { BSId, BMId, CCflag, 0,0,0 }, 0,autoflush, 0,0,0,0 };
+        LZ4F_preferences_t prefs = { 0 };
+        LZ4F_compressOptions_t options = { 0 };
         unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1;
         size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
         size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
         size_t cSize;
         U64 crcOrig, crcDecoded;
 
+        prefs.frameInfo.blockMode = BMId;
+        prefs.frameInfo.blockSizeID = BSId;
+        prefs.frameInfo.contentChecksumFlag = CCflag;
+        prefs.autoFlush = autoflush;
+
         DISPLAYUPDATE(2, "\r%5i   ", testNb);
         crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);
 
@@ -431,18 +437,21 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 size_t oSize = oend-op;
                 unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1);
                 if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
-                result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, NULL);
+                options.stableSrc = ((FUZ_rand(&randState) && 3) == 2);
+
+                result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &options);
                 CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
                 op += result;
                 ip += iSize;
+
                 if (forceFlush)
                 {
-                    result = LZ4F_flush(cCtx, op, oend-op, NULL);
+                    result = LZ4F_flush(cCtx, op, oend-op, &options);
                     CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
                     op += result;
                 }
             }
-            result = LZ4F_compressEnd(cCtx, op, oend-op, NULL);
+            result = LZ4F_compressEnd(cCtx, op, oend-op, &options);
             CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result);
             op += result;
             cSize = op-(BYTE*)compressedBuffer;