diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2014-09-12 18:28:44 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2014-09-12 18:28:44 (GMT) |
commit | a586208597f28b7b4df0913f5b95cefbd46e2a31 (patch) | |
tree | 5bc5e17df41bf65cef85d7ed272d42376cda5a0d /programs | |
parent | b1d022fa72b75af8fb373e26ac8d2f0f14a9e6fe (diff) | |
download | lz4-a586208597f28b7b4df0913f5b95cefbd46e2a31.zip lz4-a586208597f28b7b4df0913f5b95cefbd46e2a31.tar.gz lz4-a586208597f28b7b4df0913f5b95cefbd46e2a31.tar.bz2 |
lz4frame : autoflush mode : saves memory, and improves compression speed
LZ4F_getFrameInfo() fix, thanks to Christopher Jackson
Diffstat (limited to 'programs')
-rw-r--r-- | programs/frametest.c | 21 | ||||
-rw-r--r-- | programs/fullbench.c | 39 |
2 files changed, 32 insertions, 28 deletions
diff --git a/programs/frametest.c b/programs/frametest.c index c70cc2f..c3662b4 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -258,20 +258,20 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "Using 64 KB block : \n"); prefs.frameInfo.blockSizeID = max64KB; prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); + cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "without checksum : \n"); prefs.frameInfo.contentChecksumFlag = noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); + cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 256 KB block : \n"); prefs.frameInfo.blockSizeID = max256KB; prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); + cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); @@ -311,33 +311,33 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "without checksum : \n"); prefs.frameInfo.contentChecksumFlag = noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); + cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 1 MB block : \n"); prefs.frameInfo.blockSizeID = max1MB; prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); + cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "without checksum : \n"); prefs.frameInfo.contentChecksumFlag = noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); + cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 4 MB block : \n"); prefs.frameInfo.blockSizeID = max4MB; prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); + cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "without checksum : \n"); prefs.frameInfo.contentChecksumFlag = noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); + cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); @@ -404,7 +404,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi unsigned BSId = 4 + (FUZ_rand(&randState) & 3); unsigned BMId = FUZ_rand(&randState) & 1; unsigned CCflag = FUZ_rand(&randState) & 1; - LZ4F_preferences_t prefs = { { BSId, BMId, CCflag, 0,0,0 }, 0,0, 0,0,0,0 }; + unsigned autoflush = (FUZ_rand(&randState) & 3) == 2; + LZ4F_preferences_t prefs = { { BSId, BMId, CCflag, 0,0,0 }, 0,autoflush, 0,0,0,0 }; unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize); @@ -430,7 +431,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi size_t oSize = oend-op; unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1); if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - result = LZ4F_compress(cCtx, op, oSize, ip, iSize, NULL); + result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, NULL); CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); op += result; ip += iSize; diff --git a/programs/fullbench.c b/programs/fullbench.c index 96120e3..794b5b2 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -323,7 +323,7 @@ static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) { - return LZ4F_compressFrame(out, 2*inSize + (4<<20), in, inSize, NULL); + return LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL); } static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) @@ -431,22 +431,6 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) return 12; } - // Init chunks data - { - int i; - size_t remaining = benchedSize; - char* in = orig_buff; - char* out = compressed_buff; - for (i=0; i<nbChunks; i++) - { - chunkP[i].id = i; - chunkP[i].origBuffer = in; in += chunkSize; - if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } - chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; - chunkP[i].compressedSize = 0; - } - } - // Fill input buffer DISPLAY("Loading %s... \r", inFileName); readSize = fread(orig_buff, 1, benchedSize, inFile); @@ -484,6 +468,23 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue; + // Init chunks data + { + int i; + size_t remaining = benchedSize; + char* in = orig_buff; + char* out = compressed_buff; + nbChunks = (int) ((int)benchedSize / chunkSize) + 1; + for (i=0; i<nbChunks; i++) + { + chunkP[i].id = i; + chunkP[i].origBuffer = in; in += chunkSize; + if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; + chunkP[i].compressedSize = 0; + } + } + switch(cAlgNb) { case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break; @@ -499,7 +500,9 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break; case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break; case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break; - case 14: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; break; + case 14: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; + chunkP[0].origSize = benchedSize; nbChunks=1; + break; default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; } |