small improvements to lz4frame compression

4 files changed, 90 insertions, 88 deletions

diff --git a/NEWS b/NEWS
index 69e5c4e..ee4e40b 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+r123:
+Added : experimental lz4frame API, thanks to Takayuki Matsuoka and Christopher Jackson for testings
+Fix : test mode (-t) no longer requires confirmation, thanks to Thary Nguyen
+
 r122:
 Fix : AIX & AIX64 support (SamG)
 Fix : mips 64-bits support (lew van)
diff --git a/lz4frame.c b/lz4frame.c
index 7bc2319..77b96c9 100644
--- a/lz4frame.c
+++ b/lz4frame.c
@@ -245,6 +245,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
     cctxI.maxBufferSize = 64 KB;   /* mess with real buffer size, to prevent allocation; works because autoflush==1 & stableSrc==1 */
     prefs.autoFlush = 1;
     options.stableSrc = 1;
+    if (srcSize <= 64 KB) prefs.frameInfo.blockMode = blockIndependent;   /* no need for linked blocks */
 
 	if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs))
 		return -ERROR_dstMaxSize_tooSmall;
@@ -385,6 +386,27 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP
 }
 
 
+typedef int (*compressFunc_t)(void*, const char*, char*, int, int);
+
+static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx)
+{
+    /* compress one block */
+    BYTE* cSizePtr = (BYTE*)dst;
+    U32 cSize;
+    cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1));
+    LZ4F_writeLE32(cSizePtr, cSize);
+    if (cSize == 0)   /* compression failed */
+    {
+        cSize = srcSize;
+        LZ4F_writeLE32(cSizePtr, srcSize + LZ4F_BLOCKUNCOMPRESSED_FLAG);
+        memcpy(cSizePtr+4, src, srcSize);
+    }
+    return cSize + 4;
+}
+
+
+typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
+
 /* LZ4F_compressUpdate()
  * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
  * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
@@ -403,8 +425,8 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
 	const BYTE* const srcEnd = srcPtr + srcSize;
 	BYTE* const dstStart = (BYTE*)dstBuffer;
 	BYTE* dstPtr = dstStart;
-	U32 lastBlockCompressed = 0;
-	int (*compress)(void*, const char*, char*, int, int);
+	LZ4F_lastBlockStatus lastBlockCompressed = notDone;
+	compressFunc_t compress;
 
 
 	if (cctxPtr->cStage != 1) return -ERROR_GENERIC;
@@ -426,26 +448,17 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
 			memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
 			srcPtr = srcEnd;
 			cctxPtr->tmpInSize += srcSize;
+			/* still needs some CRC */
 		}
 		else
 		{
 			/* complete tmpIn block and then compress it */
-			BYTE* cSizePtr = dstPtr;
-			U32 cSize;
-			lastBlockCompressed = 1;
+			lastBlockCompressed = fromTmpBuffer;
 			memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
 			srcPtr += sizeToCopy;
-			dstPtr += 4;   /* space for cSize */
-			cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)cctxPtr->tmpIn, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1));
-			dstPtr += cSize;
-			LZ4F_writeLE32(cSizePtr, cSize);
-			if (cSize == 0)   /* compression failed : non compressible assumed */
-			{
-				cSize = blockSize + LZ4F_BLOCKUNCOMPRESSED_FLAG;
-				LZ4F_writeLE32(cSizePtr, cSize);
-				memcpy(dstPtr, cctxPtr->tmpIn, blockSize);
-				dstPtr += blockSize;
-			}
+
+			dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, &(cctxPtr->lz4ctx));
+
 			if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += blockSize;
 			cctxPtr->tmpInSize = 0;
 		}
@@ -453,59 +466,45 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
 
 	while ((size_t)(srcEnd - srcPtr) >= blockSize)
 	{
-		/* compress one block */
-		BYTE* cSizePtr = dstPtr;
-		U32 cSize;
-        lastBlockCompressed = 2;
-		dstPtr += 4;   /* space for cSizePtr */
-		cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)srcPtr, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1));
-		dstPtr += cSize;
-		LZ4F_writeLE32(cSizePtr, cSize);
-		if (cSize == 0)   /* compression failed */
-		{
-			cSize = blockSize + LZ4F_BLOCKUNCOMPRESSED_FLAG;
-			LZ4F_writeLE32(cSizePtr, cSize);
-			memcpy(dstPtr, srcPtr, blockSize);
-			dstPtr += blockSize;
-		}
+		/* compress full block */
+        lastBlockCompressed = fromSrcBuffer;
+        dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, &(cctxPtr->lz4ctx));
 		srcPtr += blockSize;
 	}
 
     if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd))
     {
-        /* compress remaining input */
-        BYTE* cSizePtr = dstPtr;
-        U32 cSize;
-        size_t iSize = srcEnd - srcPtr;
-        lastBlockCompressed = 2;
-        dstPtr += 4;   /* space for cSizePtr */
-        cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)srcPtr, (char*)dstPtr, (int)(iSize), (int)(iSize-1));
-        dstPtr += cSize;
-        LZ4F_writeLE32(cSizePtr, cSize);
-        if (cSize == 0)   /* compression failed */
-        {
-            cSize = iSize + LZ4F_BLOCKUNCOMPRESSED_FLAG;
-            LZ4F_writeLE32(cSizePtr, cSize);
-            memcpy(dstPtr, srcPtr, iSize);
-            dstPtr += iSize;
-        }
-        srcPtr += iSize;
+        /* compress remaining input < blockSize */
+        lastBlockCompressed = fromSrcBuffer;
+        dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, &(cctxPtr->lz4ctx));
+		srcPtr  = srcEnd;
     }
 
-    /* save last input up to 64 KB for dictionary */
-	if ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) && (lastBlockCompressed) && (!compressOptionsPtr->stableSrc))
+    /* preserve dictionary if necessary */
+    if ((cctxPtr->prefs.frameInfo.blockMode==blockLinked) && (lastBlockCompressed==fromSrcBuffer))
     {
-        if ((lastBlockCompressed==2) ||
-           ((cctxPtr->tmpBuff + cctxPtr->maxBufferSize) < (cctxPtr->tmpIn + cctxPtr->maxBlockSize)))
+        if (compressOptionsPtr->stableSrc)
         {
-            int result;
-            result = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB);
-            if (result==0) return -ERROR_GENERIC;
-            cctxPtr->tmpIn = cctxPtr->tmpBuff + result;
+            cctxPtr->tmpIn = cctxPtr->tmpBuff;
         }
+        else
+        {
+            int realDictSize;
+            realDictSize = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB);
+            if (realDictSize==0) return -ERROR_GENERIC;
+            cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+        }
+    }
+
+    /* keep tmpIn within limits */
+    if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize))   /* necessarily blockLinked && lastBlockCompressed==fromTmpBuffer */
+    {
+        LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB);
+        cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB;
     }
 
-	if (srcPtr < srcEnd)   /* some input data left, necessarily < blockSize */
+    /* some input data left, necessarily < blockSize */
+	if (srcPtr < srcEnd)
     {
         /* fill tmp buffer */
         size_t sizeToCopy = srcEnd - srcPtr;
@@ -547,31 +546,16 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer,
             (int(*)(void*,const char*,char*,int,int))LZ4_compress_limitedOutput_continue :
             LZ4_compress_limitedOutput_withState;
 
-	{
-		BYTE* cSizePtr = dstPtr;
-		U32 cSize;
-		dstPtr += 4;   /* space for cSizePtr */
-		cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)cctxPtr->tmpIn, (char*)dstPtr, (int)(cctxPtr->tmpInSize), (int)(cctxPtr->tmpInSize-1));
-		dstPtr += cSize;
-		LZ4F_writeLE32(cSizePtr, cSize);
-		if (cSize == 0)   /* compression failed */
-		{
-			cSize = cctxPtr->tmpInSize + LZ4F_BLOCKUNCOMPRESSED_FLAG;
-			LZ4F_writeLE32(cSizePtr, cSize);
-			memcpy(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize);
-			dstPtr += cctxPtr->tmpInSize;
-		}
-        if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
-		cctxPtr->tmpInSize = 0;
-	}
+    /* compress tmp buffer */
+    dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, &(cctxPtr->lz4ctx));
+    if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
+	cctxPtr->tmpInSize = 0;
 
-	if ((cctxPtr->prefs.frameInfo.blockMode == blockLinked)
-       && ((cctxPtr->tmpBuff + cctxPtr->maxBufferSize) < (cctxPtr->tmpIn + cctxPtr->maxBlockSize)))
+    /* keep tmpIn within limits */
+    if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize))   /* necessarily blockLinked */
     {
-        /* last 64 KB of input become dictionary */
-        int result = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB);
-        if (!result) return ERROR_GENERIC;
-        cctxPtr->tmpIn = cctxPtr->tmpBuff + result;
+        LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB);
+        cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB;
     }
 
 	return dstPtr - dstStart;
@@ -1114,8 +1098,20 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
         }
     }
 
-    if ((dctxPtr->frameInfo.blockMode==blockLinked) && (dctxPtr->dict != dctxPtr->tmpOutBuffer))
+    if ( (dctxPtr->frameInfo.blockMode==blockLinked)
+       &&(dctxPtr->dict != dctxPtr->tmpOutBuffer)
+        )
         LZ4F_saveDict(dctxPtr, NULL, 0);
+        //(!decompressOptionsPtr->stableDst + 1) )
+    /*{
+        size_t newDictSize = dctxPtr->dictSize;
+        BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize;
+        if ((newDictSize) > 64 KB) newDictSize = 64 KB;
+        memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
+        dctxPtr->dict = dctxPtr->tmpOutBuffer;
+        dctxPtr->dictSize = newDictSize;
+        dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize;
+    }*/
 
     if (srcPtr<srcEnd)   /* function must be called again with following source data */
         dctxPtr->srcExpect = srcPtr;
diff --git a/lz4frame.h b/lz4frame.h
index 0769f6a..4bc8c59 100644
--- a/lz4frame.h
+++ b/lz4frame.h
@@ -187,7 +187,7 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB
  * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
  * The function outputs an error code if it fails (can be tested using LZ4F_isError())
  * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
- * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
+ * compressionContext can then be used again, starting with LZ4F_compressBegin().
  */
 
 
diff --git a/programs/frametest.c b/programs/frametest.c
index 4d09411..9f14bfb 100644
--- a/programs/frametest.c
+++ b/programs/frametest.c
@@ -406,7 +406,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
         unsigned CCflag = FUZ_rand(&randState) & 1;
         unsigned autoflush = (FUZ_rand(&randState) & 7) == 2;
         LZ4F_preferences_t prefs = { 0 };
-        LZ4F_compressOptions_t options = { 0 };
+        LZ4F_compressOptions_t cOptions = { 0 };
+        LZ4F_decompressOptions_t dOptions = { 0 };
         unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1;
         size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
         size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
@@ -437,21 +438,21 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 size_t oSize = oend-op;
                 unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1);
                 if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
-                options.stableSrc = ((FUZ_rand(&randState) && 3) == 2);
+                cOptions.stableSrc = ((FUZ_rand(&randState) && 3) == 1);
 
-                result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &options);
+                result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
                 CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
                 op += result;
                 ip += iSize;
 
                 if (forceFlush)
                 {
-                    result = LZ4F_flush(cCtx, op, oend-op, &options);
+                    result = LZ4F_flush(cCtx, op, oend-op, &cOptions);
                     CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
                     op += result;
                 }
             }
-            result = LZ4F_compressEnd(cCtx, op, oend-op, &options);
+            result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions);
             CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result);
             op += result;
             cSize = op-(BYTE*)compressedBuffer;
@@ -471,7 +472,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2;
                 if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
                 if (oSize > (size_t)(oend-op)) oSize = oend-op;
-                result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
+                dOptions.stableDst = FUZ_rand(&randState) & 1;
+                result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
                 if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
                 CHECK(LZ4F_isError(result), "Decompression failed (error %i)", (int)result);
                 op += oSize;