lz4frame : autoflush mode : saves memory, and improves compression speed

LZ4F_getFrameInfo() fix, thanks to Christopher Jackson

2 files changed, 32 insertions, 28 deletions

diff --git a/programs/frametest.c b/programs/frametest.c
index c70cc2f..c3662b4 100644
--- a/programs/frametest.c
+++ b/programs/frametest.c
@@ -258,20 +258,20 @@ int basicTests(U32 seed, double compressibility)
 	DISPLAYLEVEL(3, "Using 64 KB block : \n");
 	prefs.frameInfo.blockSizeID = max64KB;
 	prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
-	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
+	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
 	if (LZ4F_isError(cSize)) goto _output_error;
 	DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
 	DISPLAYLEVEL(3, "without checksum : \n");
 	prefs.frameInfo.contentChecksumFlag = noContentChecksum;
-	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
+	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
 	if (LZ4F_isError(cSize)) goto _output_error;
 	DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
 	DISPLAYLEVEL(3, "Using 256 KB block : \n");
 	prefs.frameInfo.blockSizeID = max256KB;
 	prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
-	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
+	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
 	if (LZ4F_isError(cSize)) goto _output_error;
 	DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
@@ -311,33 +311,33 @@ int basicTests(U32 seed, double compressibility)
 
 	DISPLAYLEVEL(3, "without checksum : \n");
 	prefs.frameInfo.contentChecksumFlag = noContentChecksum;
-	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
+	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
 	if (LZ4F_isError(cSize)) goto _output_error;
 	DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
 	DISPLAYLEVEL(3, "Using 1 MB block : \n");
 	prefs.frameInfo.blockSizeID = max1MB;
 	prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
-	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
+	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
 	if (LZ4F_isError(cSize)) goto _output_error;
 	DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
 	DISPLAYLEVEL(3, "without checksum : \n");
 	prefs.frameInfo.contentChecksumFlag = noContentChecksum;
-	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
+	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
 	if (LZ4F_isError(cSize)) goto _output_error;
 	DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
 	DISPLAYLEVEL(3, "Using 4 MB block : \n");
 	prefs.frameInfo.blockSizeID = max4MB;
 	prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
-	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
+	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
 	if (LZ4F_isError(cSize)) goto _output_error;
 	DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
 	DISPLAYLEVEL(3, "without checksum : \n");
 	prefs.frameInfo.contentChecksumFlag = noContentChecksum;
-	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
+	cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
 	if (LZ4F_isError(cSize)) goto _output_error;
 	DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
@@ -404,7 +404,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
         unsigned BSId   = 4 + (FUZ_rand(&randState) & 3);
         unsigned BMId   = FUZ_rand(&randState) & 1;
         unsigned CCflag = FUZ_rand(&randState) & 1;
-        LZ4F_preferences_t prefs = { { BSId, BMId, CCflag, 0,0,0 }, 0,0, 0,0,0,0 };
+        unsigned autoflush = (FUZ_rand(&randState) & 3) == 2;
+        LZ4F_preferences_t prefs = { { BSId, BMId, CCflag, 0,0,0 }, 0,autoflush, 0,0,0,0 };
         unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1;
         size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
         size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
@@ -430,7 +431,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 size_t oSize = oend-op;
                 unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1);
                 if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
-                result = LZ4F_compress(cCtx, op, oSize, ip, iSize, NULL);
+                result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, NULL);
                 CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
                 op += result;
                 ip += iSize;
diff --git a/programs/fullbench.c b/programs/fullbench.c
index 96120e3..794b5b2 100644
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -323,7 +323,7 @@ static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out
 
 static int local_LZ4F_compressFrame(const char* in, char* out, int inSize)
 {
-    return LZ4F_compressFrame(out, 2*inSize + (4<<20), in, inSize, NULL);
+    return LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL);
 }
 
 static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
@@ -431,22 +431,6 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
         return 12;
       }
 
-      // Init chunks data
-      {
-          int i;
-          size_t remaining = benchedSize;
-          char* in = orig_buff;
-          char* out = compressed_buff;
-          for (i=0; i<nbChunks; i++)
-          {
-              chunkP[i].id = i;
-              chunkP[i].origBuffer = in; in += chunkSize;
-              if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
-              chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
-              chunkP[i].compressedSize = 0;
-          }
-      }
-
       // Fill input buffer
       DISPLAY("Loading %s...       \r", inFileName);
       readSize = fread(orig_buff, 1, benchedSize, inFile);
@@ -484,6 +468,23 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
 
             if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue;
 
+            // Init chunks data
+            {
+              int i;
+              size_t remaining = benchedSize;
+              char* in = orig_buff;
+              char* out = compressed_buff;
+              nbChunks = (int) ((int)benchedSize / chunkSize) + 1;
+              for (i=0; i<nbChunks; i++)
+              {
+                  chunkP[i].id = i;
+                  chunkP[i].origBuffer = in; in += chunkSize;
+                  if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
+                  chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
+                  chunkP[i].compressedSize = 0;
+              }
+            }
+
             switch(cAlgNb)
             {
             case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break;
@@ -499,7 +500,9 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
             case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break;
             case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
             case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
-            case 14: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; break;
+            case 14: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame";
+                        chunkP[0].origSize = benchedSize; nbChunks=1;
+                        break;
             default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
             }