diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2015-05-07 11:23:39 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2015-05-07 11:23:39 (GMT) |
commit | 7d182b816ace89d6d6d16b7aee376a9962a05caa (patch) | |
tree | 5b2236b59cd0e42e3365b0765041a25c85519b39 /programs | |
parent | 160661c7a4cbf805f4af74d2e3932a17a66e6ce7 (diff) | |
parent | fdd0029c3778768d0e8a80ce238d981f946c8b78 (diff) | |
download | lz4-7d182b816ace89d6d6d16b7aee376a9962a05caa.zip lz4-7d182b816ace89d6d6d16b7aee376a9962a05caa.tar.gz lz4-7d182b816ace89d6d6d16b7aee376a9962a05caa.tar.bz2 |
Dev
Diffstat (limited to 'programs')
-rw-r--r-- | programs/Makefile | 118 | ||||
-rw-r--r-- | programs/bench.c | 31 | ||||
-rw-r--r-- | programs/frametest.c | 152 | ||||
-rw-r--r-- | programs/fullbench.c | 454 | ||||
-rw-r--r-- | programs/fuzzer.c | 269 | ||||
-rw-r--r-- | programs/lz4.1 | 14 | ||||
-rw-r--r-- | programs/lz4cli.c | 72 | ||||
-rw-r--r-- | programs/lz4io.c | 672 | ||||
-rw-r--r-- | programs/lz4io.h | 3 |
9 files changed, 1119 insertions, 666 deletions
diff --git a/programs/Makefile b/programs/Makefile index a324148..39335db 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -27,24 +27,24 @@ # lz4c32: Same as lz4c, but forced to compile in 32-bits mode # fuzzer : Test tool, to check lz4 integrity on target platform # fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode +# frametest : Test tool, to check lz4frame integrity on target platform +# frametest32: Same as frametest, but forced to compile in 32-bits mode # fullbench : Precisely measure speed for each LZ4 function variant # fullbench32: Same as fullbench, but forced to compile in 32-bits mode +# datagen : generates synthetic data samples for tests & benchmarks # ########################################################################## -RELEASE?= r128 +RELEASE?= r129 DESTDIR?= PREFIX ?= /usr/local CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\" -FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\" +FLAGS := -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -BINDIR=$(PREFIX)/bin -MANDIR=$(PREFIX)/share/man/man1 -LZ4DIR=../lib - -TEST_FILES = COPYING -TEST_TARGETS=test-native +BINDIR := $(PREFIX)/bin +MANDIR := $(PREFIX)/share/man/man1 +LZ4DIR := ../lib # Define *.exe as extension for Windows systems @@ -58,7 +58,10 @@ endif # Select test target for Travis CI's Build Matrix -TRAVIS_TARGET=$(LZ4_TRAVIS_CI_ENV) +TRAVIS_TARGET:= $(LZ4_TRAVIS_CI_ENV) +TEST_FILES := COPYING +TEST_TARGETS := test-native +FUZZER_TIME := -T9mn default: lz4 @@ -100,7 +103,7 @@ datagen : datagen.c datagencli.c $(CC) $(FLAGS) $^ -o $@$(EXT) clean: - @rm -f core *.o *.test \ + @rm -f core *.o *.test tmp* \ lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) \ @@ -148,15 +151,18 @@ test-travis: $(TRAVIS_TARGET) test-lz4-sparse: lz4 datagen @echo ---- test sparse file support ---- - ./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dv --sparse > tmpB4 - ./datagen -g50M -P100 | ./lz4 -B5D | ./lz4 -dv --sparse > tmpB5 - ./datagen -g50M -P100 | ./lz4 -B6D | ./lz4 -dv --sparse > tmpB6 - ./datagen -g50M -P100 | ./lz4 -B7D | ./lz4 -dv --sparse > tmpB7 + ./datagen -g5M -P100 > tmpSrc + ./lz4 -B4D tmpSrc | ./lz4 -dv --sparse > tmpB4 + diff -s tmpSrc tmpB4 + ./lz4 -B5D tmpSrc | ./lz4 -dv --sparse > tmpB5 + diff -s tmpSrc tmpB5 + ./lz4 -B6D tmpSrc | ./lz4 -dv --sparse > tmpB6 + diff -s tmpSrc tmpB6 + ./lz4 -B7D tmpSrc | ./lz4 -dv --sparse > tmpB7 + diff -s tmpSrc tmpB7 + ./lz4 tmpSrc | ./lz4 -dv --no-sparse > tmpNoSparse + diff -s tmpSrc tmpNoSparse ls -ls tmp* - ./datagen -g50M -P100 | diff -s - tmpB4 - ./datagen -g50M -P100 | diff -s - tmpB5 - ./datagen -g50M -P100 | diff -s - tmpB6 - ./datagen -g50M -P100 | diff -s - tmpB7 ./datagen -s1 -g1200007 -P100 | ./lz4 | ./lz4 -dv --sparse > tmpOdd # Odd size file (to not finish on an exact nb of blocks) ./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd ls -ls tmpOdd @@ -168,6 +174,11 @@ test-lz4-contentSize: lz4 datagen ./lz4 -v tmp | ./lz4 -t ./lz4 -v --content-size tmp | ./lz4 -d > tmp2 diff -s tmp tmp2 + # test large size [2-4] GB + @./datagen -g3G -P100 | ./lz4 | ./lz4 --decompress --force --sparse - tmp + @ls -ls tmp + ./lz4 --quiet --content-size tmp | ./lz4 --verbose --decompress --force --sparse - tmp2 + @ls -ls tmp2 @rm tmp* test-lz4-frame-concatenation: lz4 datagen @@ -183,24 +194,30 @@ test-lz4-frame-concatenation: lz4 datagen @rm *.test @echo frame concatenation test completed -test-lz4: lz4 datagen test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation +test-lz4-multiple: lz4 datagen + @echo ---- test multiple files ---- + @./datagen -s1 > tmp1 2> $(VOID) + @./datagen -s2 -g100K > tmp2 2> $(VOID) + @./datagen -s3 -g1M > tmp3 2> $(VOID) + ./lz4 -f -m tmp* + ls -ls tmp* + rm tmp1 tmp2 tmp3 + ./lz4 -df -m *.lz4 + ls -ls tmp* + ./lz4 -f -m tmp1 notHere tmp2; echo $$? + @rm tmp* + +test-lz4: lz4 datagen test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation @echo ---- test lz4 basic compression/decompression ---- ./datagen -g0 | ./lz4 -v | ./lz4 -t ./datagen -g16KB | ./lz4 -9 | ./lz4 -t ./datagen | ./lz4 | ./lz4 -t ./datagen -g6M -P99 | ./lz4 -9BD | ./lz4 -t - ./datagen -g17M | ./lz4 -9v | ./lz4 -tq + ./datagen -g17M | ./lz4 -9v | ./lz4 -qt ./datagen -g33M | ./lz4 --no-frame-crc | ./lz4 -t ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t - ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -t - ./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -t - @echo ---- test multiple input files ---- - @./datagen -s1 > file1 - @./datagen -s2 > file2 - @./datagen -s3 > file3 - ./lz4 -f -m file1 file2 file3 - ls -l file* - @rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4 + ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -qt + ./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -qt @echo ---- test pass-through ---- ./datagen | ./lz4 -tf @@ -212,41 +229,46 @@ test-lz4c32: lz4 lz4c32 datagen ./datagen -g16KB | ./lz4c32 -9 | ./lz4 -t ./datagen | ./lz4c32 | ./lz4c32 -t ./datagen | ./lz4c32 | ./lz4 -t - ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -t - ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -t - ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -t - ./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -t + ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -qt + ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -qt + ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -qt + ./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -qt test-fullbench: fullbench - ./fullbench --no-prompt $(TEST_FILES) + ./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES) test-fullbench32: fullbench32 - ./fullbench32 --no-prompt $(TEST_FILES) + ./fullbench32 --no-prompt $(NB_LOOPS) $(TEST_FILES) test-fuzzer: fuzzer - ./fuzzer + ./fuzzer $(FUZZER_TIME) test-fuzzer32: fuzzer32 - ./fuzzer32 + ./fuzzer32 $(FUZZER_TIME) test-frametest: frametest - ./frametest + ./frametest $(FUZZER_TIME) test-frametest32: frametest32 - ./frametest32 + ./frametest32 $(FUZZER_TIME) -test-mem: lz4 datagen fuzzer frametest - valgrind --leak-check=yes ./datagen -g50M > $(VOID) +test-mem: lz4 datagen fuzzer frametest fullbench + valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) ./datagen -g16KB > tmp - valgrind --leak-check=yes ./lz4 -9 -BD -f tmp $(VOID) + valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -BD -f tmp $(VOID) + ./datagen -g16KB -s2 > tmp2 + ./datagen -g16KB -s3 > tmp3 + valgrind --leak-check=yes --error-exitcode=1 ./lz4 --force --multiple tmp tmp2 tmp3 ./datagen -g16MB > tmp - valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp tmp2 - valgrind --leak-check=yes ./lz4 -t tmp2 + valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -B5D -f tmp tmp2 + valgrind --leak-check=yes --error-exitcode=1 ./lz4 -t tmp2 + valgrind --leak-check=yes --error-exitcode=1 ./lz4 -bi1 tmp + valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2 ./datagen -g256MB > tmp - valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp $(VOID) + valgrind --leak-check=yes --error-exitcode=1 ./lz4 -B4D -f -vq tmp $(VOID) rm tmp* - valgrind --leak-check=yes ./fuzzer -i64 -t1 - valgrind --leak-check=yes ./frametest -i256 + valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1 + valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256 test-mem32: lz4c32 datagen # unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system... diff --git a/programs/bench.c b/programs/bench.c index e1b5357..9f949c4 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -58,9 +58,9 @@ #include "lz4.h" #define COMPRESSOR0 LZ4_compress_local -static int LZ4_compress_local(const char* src, char* dst, int size, int clevel) { (void)clevel; return LZ4_compress(src, dst, size); } +static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSize, int clevel) { (void)clevel; return LZ4_compress_default(src, dst, srcSize, dstSize); } #include "lz4hc.h" -#define COMPRESSOR1 LZ4_compressHC2 +#define COMPRESSOR1 LZ4_compress_HC #define DEFAULTCOMPRESSOR COMPRESSOR0 #include "xxhash.h" @@ -121,8 +121,8 @@ struct chunkParameters struct compressionParameters { - int (*compressionFunction)(const char*, char*, int, int); - int (*decompressionFunction)(const char*, char*, int); + int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); + int (*decompressionFunction)(const char* src, char* dst, int dstSize); }; @@ -281,17 +281,13 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) /* Check file existence */ inFileName = fileNamesTable[fileIdx++]; inFile = fopen( inFileName, "rb" ); - if (inFile==NULL) - { - DISPLAY( "Pb opening %s\n", inFileName); - return 11; - } + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } /* Memory allocation & restrictions */ inFileSize = BMK_GetFileSize(inFileName); - if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; } + if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; } benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2; - if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; } + if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; } if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) { @@ -306,7 +302,6 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) compressedBuffSize = nbChunks * maxCompressedChunkSize; compressedBuffer = (char*)malloc((size_t)compressedBuffSize); - if (!orig_buff || !compressedBuffer) { DISPLAY("\nError: not enough memory!\n"); @@ -376,11 +371,12 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { for (chunkNb=0; chunkNb<nbChunks; chunkNb++) - chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, cLevel); + chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize, cLevel); nbLoops++; } milliTime = BMK_GetMilliSpan(milliTime); + nbLoops += !nbLoops; /* avoid division by zero */ if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime/nbLoops; cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; ratio = (double)cSize/(double)benchedSize*100.; @@ -402,8 +398,9 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) } milliTime = BMK_GetMilliSpan(milliTime); + nbLoops += !nbLoops; /* avoid division by zero */ if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime/nbLoops; - DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); /* CRC Checking */ crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0); @@ -413,9 +410,9 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) if (crcOrig==crcCheck) { if (ratio<100.) - DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); else - DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); } totals += benchedSize; totalz += cSize; @@ -431,7 +428,7 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) if (nbFiles > 1) DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.); - if (BMK_pause) { DISPLAY("\npress enter...\n"); getchar(); } + if (BMK_pause) { DISPLAY("\npress enter...\n"); (void)getchar(); } return 0; } diff --git a/programs/frametest.c b/programs/frametest.c index 4beee23..46ec030 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -238,10 +238,11 @@ int basicTests(U32 seed, double compressibility) U32 randState = seed; size_t cSize, testSize; LZ4F_preferences_t prefs; - LZ4F_decompressionContext_t dCtx; + LZ4F_decompressionContext_t dCtx = NULL; + LZ4F_compressionContext_t cctx = NULL; U64 crcOrig; - // Create compressible test buffer + /* Create compressible test buffer */ memset(&prefs, 0, sizeof(prefs)); CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL)); @@ -249,7 +250,7 @@ int basicTests(U32 seed, double compressibility) FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - // Trivial tests : one-step frame + /* Trivial tests : one-step frame */ testSize = COMPRESSIBLE_NOISE_LENGTH; DISPLAYLEVEL(3, "Using NULL preferences : \n"); cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); @@ -276,12 +277,58 @@ int basicTests(U32 seed, double compressibility) if (crcDest != crcOrig) goto _output_error; DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); + DISPLAYLEVEL(4, "Reusing decompression context \n"); + { + size_t iSize = compressedBufferSize - 4; + const BYTE* cBuff = (const BYTE*) compressedBuffer; + DISPLAYLEVEL(3, "Missing last 4 bytes : "); + errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + if (!errorCode) goto _output_error; + DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned)errorCode); + cBuff += iSize; + iSize = errorCode; + errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); + if (errorCode != 0) goto _output_error; + crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + } + + { + size_t oSize = 0; + size_t iSize = 0; + LZ4F_frameInfo_t fi; + + DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); + errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); + + DISPLAYLEVEL(3, "get FrameInfo on null input : "); + errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); + if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; + DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); + + DISPLAYLEVEL(3, "get FrameInfo on not enough input : "); + iSize = 6; + errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); + if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; + DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); + ip += iSize; + + DISPLAYLEVEL(3, "get FrameInfo on enough input : "); + iSize = 15 - iSize; + errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); + if (LZ4F_isError(errorCode)) goto _output_error; + DISPLAYLEVEL(3, " correctly decoded \n"); + ip += iSize; + } + DISPLAYLEVEL(3, "Byte after byte : \n"); while (ip < iend) { size_t oSize = oend-op; size_t iSize = 1; - //DISPLAY("%7i \n", (int)(ip-(BYTE*)compressedBuffer)); errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); if (LZ4F_isError(errorCode)) goto _output_error; op += oSize; @@ -289,28 +336,28 @@ int basicTests(U32 seed, double compressibility) } crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); + DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-(BYTE*)decodedBuffer), COMPRESSIBLE_NOISE_LENGTH); errorCode = LZ4F_freeDecompressionContext(dCtx); if (LZ4F_isError(errorCode)) goto _output_error; } DISPLAYLEVEL(3, "Using 64 KB block : \n"); - prefs.frameInfo.blockSizeID = max64KB; - prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; + prefs.frameInfo.blockSizeID = LZ4F_max64KB; + prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = noContentChecksum; + prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 256 KB block : \n"); - prefs.frameInfo.blockSizeID = max256KB; - prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; + prefs.frameInfo.blockSizeID = LZ4F_max256KB; + prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); @@ -350,33 +397,33 @@ int basicTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = noContentChecksum; + prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 1 MB block : \n"); - prefs.frameInfo.blockSizeID = max1MB; - prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; + prefs.frameInfo.blockSizeID = LZ4F_max1MB; + prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = noContentChecksum; + prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 4 MB block : \n"); - prefs.frameInfo.blockSizeID = max4MB; - prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; + prefs.frameInfo.blockSizeID = LZ4F_max4MB; + prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = noContentChecksum; + prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); @@ -385,7 +432,6 @@ int basicTests(U32 seed, double compressibility) size_t errorCode; BYTE* const ostart = (BYTE*)compressedBuffer; BYTE* op = ostart; - LZ4F_compressionContext_t cctx; errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) goto _output_error; @@ -430,6 +476,7 @@ int basicTests(U32 seed, double compressibility) errorCode = LZ4F_freeCompressionContext(cctx); if (LZ4F_isError(errorCode)) goto _output_error; + cctx = NULL; } DISPLAYLEVEL(3, "Skippable frame test : \n"); @@ -500,10 +547,6 @@ int basicTests(U32 seed, double compressibility) ip += iSize; } DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); - - /* release memory */ - errorCode = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(errorCode)) goto _output_error; } DISPLAY("Basic tests completed \n"); @@ -511,6 +554,8 @@ _end: free(CNBuffer); free(compressedBuffer); free(decodedBuffer); + LZ4F_freeDecompressionContext(dCtx); dCtx = NULL; + LZ4F_freeCompressionContext(cctx); cctx = NULL; return testResult; _output_error: @@ -523,8 +568,8 @@ _output_error: static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous) { int p=0; - BYTE* b1=(BYTE*)buff1; - BYTE* b2=(BYTE*)buff2; + const BYTE* b1=(const BYTE*)buff1; + const BYTE* b2=(const BYTE*)buff2; if (nonContiguous) { DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size); @@ -537,7 +582,7 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un static const U32 srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ -int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility) +int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration) { unsigned testResult = 0; unsigned testNb = 0; @@ -548,10 +593,15 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi LZ4F_decompressionContext_t dCtx = NULL; LZ4F_compressionContext_t cCtx = NULL; size_t result; + const U32 startTime = FUZ_GetMilliStart(); XXH64_state_t xxh64; # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } + + /* Init */ + duration *= 1000; + /* Create buffers */ result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); @@ -566,10 +616,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); /* jump to requested testNb */ - for (testNb =0; testNb < startTest; testNb++) (void)FUZ_rand(&coreRand); // sync randomizer + for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer /* main fuzzer test loop */ - for ( ; testNb < nbTests; testNb++) + for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++) { U32 randState = coreRand ^ prime1; unsigned BSId = 4 + (FUZ_rand(&randState) & 3); @@ -591,9 +641,9 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi memset(&prefs, 0, sizeof(prefs)); memset(&cOptions, 0, sizeof(cOptions)); memset(&dOptions, 0, sizeof(dOptions)); - prefs.frameInfo.blockMode = (blockMode_t)BMId; - prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId; - prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag; + prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId; + prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId; + prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag; prefs.frameInfo.contentSize = frameContentSize; prefs.autoFlush = autoflush; prefs.compressionLevel = FUZ_rand(&randState) % 5; @@ -662,6 +712,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1; nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */ XXH64_reset(&xxh64, 1); + if (maxBits < 3) maxBits = 3; while (ip < iend) { unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; @@ -673,7 +724,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi dOptions.stableDst = FUZ_rand(&randState) & 1; if (nonContiguousDst==2) dOptions.stableDst = 0; result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); - if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); + if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid) + locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result)); XXH64_update(&xxh64, op, (U32)oSize); totalOut += oSize; @@ -704,7 +756,7 @@ _end: if (pause) { DISPLAY("press enter to finish \n"); - getchar(); + (void)getchar(); } return testResult; @@ -721,6 +773,7 @@ int FUZ_usage(void) DISPLAY( "\n"); DISPLAY( "Arguments :\n"); DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); + DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); DISPLAY( " -s# : Select seed (default:prompt user)\n"); DISPLAY( " -t# : Select starting test number (default:0)\n"); DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); @@ -739,6 +792,7 @@ int main(int argc, char** argv) int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; int result=0; + U32 duration=0; /* Check command line */ programName = argv[0]; @@ -746,9 +800,9 @@ int main(int argc, char** argv) { char* argument = argv[argNb]; - if(!argument) continue; // Protection if argument empty + if(!argument) continue; /* Protection if argument empty */ - // Decode command (note : aggregated commands are allowed) + /* Decode command (note : aggregated commands are allowed) */ if (argument[0]=='-') { if (!strcmp(argument, "--no-prompt")) @@ -781,7 +835,7 @@ int main(int argc, char** argv) case 'i': argument++; - nbTests=0; + nbTests=0; duration=0; while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; @@ -789,6 +843,32 @@ int main(int argc, char** argv) argument++; } break; + + case 'T': + argument++; + nbTests = 0; duration = 0; + for (;;) + { + switch(*argument) + { + case 'm': duration *= 60; argument++; continue; + case 's': + case 'n': argument++; continue; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': duration *= 10; duration += *argument++ - '0'; continue; + } + break; + } + break; + case 's': argument++; seed=0; @@ -841,5 +921,5 @@ int main(int argc, char** argv) if (testNb==0) result = basicTests(seed, ((double)proba) / 100); if (result) return 1; - return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100); + return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration); } diff --git a/programs/fullbench.c b/programs/fullbench.c index 0c6e05e..0d08a40 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -30,7 +30,7 @@ #define _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -// Unix Large Files support (>4GB) +/* Unix Large Files support (>4GB) */ #if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions # define _LARGEFILE_SOURCE # define _FILE_OFFSET_BITS 64 @@ -47,17 +47,17 @@ /************************************** * Includes **************************************/ -#include <stdlib.h> // malloc -#include <stdio.h> // fprintf, fopen, ftello64 -#include <sys/types.h> // stat64 -#include <sys/stat.h> // stat64 -#include <string.h> // strcmp +#include <stdlib.h> /* malloc, free */ +#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <sys/types.h> /* stat64 */ +#include <sys/stat.h> /* stat64 */ +#include <string.h> /* strcmp */ -// Use ftime() if gettimeofday() is not available on your target +/* Use ftime() if gettimeofday() is not available on your target */ #if defined(BMK_LEGACY_TIMER) -# include <sys/timeb.h> // timeb, ftime +# include <sys/timeb.h> /* timeb, ftime */ #else -# include <sys/time.h> // gettimeofday +# include <sys/time.h> /* gettimeofday */ #endif #include "lz4.h" @@ -75,16 +75,11 @@ # define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) #endif -// GCC does not support _rotl outside of Windows -#if !defined(_WIN32) -# define _rotl(x,r) ((x << r) | (x >> (32 - r))) -#endif - /************************************** * Basic Types **************************************/ -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include <stdint.h> typedef uint8_t BYTE; typedef uint16_t U16; @@ -118,7 +113,7 @@ #define GB *(1U<<30) #define KNUTH 2654435761U -#define MAX_MEM (1984 MB) +#define MAX_MEM (1920 MB) #define DEFAULT_CHUNKSIZE (4 MB) #define ALL_COMPRESSORS 0 @@ -139,53 +134,53 @@ struct chunkParameters /************************************** -* MACRO +* Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define PROGRESS(...) no_prompt ? 0 : DISPLAY(__VA_ARGS__) - +#define PROGRESS(...) g_noPrompt ? 0 : DISPLAY(__VA_ARGS__) -//************************************** -// Benchmark Parameters -//************************************** -static int chunkSize = DEFAULT_CHUNKSIZE; -static int nbIterations = NBLOOPS; -static int BMK_pause = 0; -static int compressionTest = 1; -static int decompressionTest = 1; -static int compressionAlgo = ALL_COMPRESSORS; -static int decompressionAlgo = ALL_DECOMPRESSORS; -static int no_prompt = 0; +/************************************** +* Benchmark Parameters +**************************************/ +static int g_chunkSize = DEFAULT_CHUNKSIZE; +static int g_nbIterations = NBLOOPS; +static int g_pause = 0; +static int g_compressionTest = 1; +static int g_compressionAlgo = ALL_COMPRESSORS; +static int g_decompressionTest = 1; +static int g_decompressionAlgo = ALL_DECOMPRESSORS; +static int g_noPrompt = 0; -void BMK_SetBlocksize(int bsize) +static void BMK_setBlocksize(int bsize) { - chunkSize = bsize; - DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10); + g_chunkSize = bsize; + DISPLAY("-Using Block Size of %i KB-\n", g_chunkSize>>10); } -void BMK_SetNbIterations(int nbLoops) +static void BMK_setNbIterations(int nbLoops) { - nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", nbIterations); + g_nbIterations = nbLoops; + DISPLAY("- %i iterations -\n", g_nbIterations); } -void BMK_SetPause(void) +static void BMK_setPause(void) { - BMK_pause = 1; + g_pause = 1; } -//********************************************************* -// Private functions -//********************************************************* + +/********************************************************* +* Private functions +*********************************************************/ #if defined(BMK_LEGACY_TIMER) static int BMK_GetMilliStart(void) { - // Based on Legacy ftime() - // Rolls over every ~ 12.1 days (0x100000/24/60/60) - // Use GetMilliSpan to correct for rollover + /* Based on Legacy ftime() + * Rolls over every ~ 12.1 days (0x100000/24/60/60) + * Use GetMilliSpan to correct for rollover */ struct timeb tb; int nCount; ftime( &tb ); @@ -197,8 +192,8 @@ static int BMK_GetMilliStart(void) static int BMK_GetMilliStart(void) { - // Based on newer gettimeofday() - // Use GetMilliSpan to correct for rollover + /* Based on newer gettimeofday() + * Use GetMilliSpan to correct for rollover */ struct timeval tv; int nCount; gettimeofday(&tv, NULL); @@ -253,7 +248,7 @@ static U64 BMK_GetFileSize(char* infilename) struct stat statbuf; r = stat(infilename, &statbuf); #endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ return (U64)statbuf.st_size; } @@ -381,106 +376,141 @@ start: #endif // __SSSE3__ -static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) +static LZ4_stream_t LZ4_stream; +static void local_LZ4_resetDictT(void) { - return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); + LZ4_resetStream(&LZ4_stream); } -static void* stateLZ4; -static int local_LZ4_compress_withState(const char* in, char* out, int inSize) +static void local_LZ4_createStream(void) { - return LZ4_compress_withState(stateLZ4, in, out, inSize); + LZ4_resetStream(&LZ4_stream); } -static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize) +static int local_LZ4_saveDict(const char* in, char* out, int inSize) { - return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize)); + (void)in; + return LZ4_saveDict(&LZ4_stream, out, inSize); } -static LZ4_stream_t* ctx; -static int local_LZ4_compress_continue(const char* in, char* out, int inSize) +static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) { - return LZ4_compress_continue(ctx, in, out, inSize); + return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1); } -static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize) +static int local_LZ4_compress_default_large(const char* in, char* out, int inSize) { - return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)); } +static int local_LZ4_compress_default_small(const char* in, char* out, int inSize) +{ + return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1); +} -LZ4_stream_t LZ4_dict; -static void* local_LZ4_resetDictT(const char* fake) +static int local_LZ4_compress_fast0(const char* in, char* out, int inSize) { - (void)fake; - memset(&LZ4_dict, 0, sizeof(LZ4_stream_t)); - return NULL; + return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 0); } -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize); -static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize) +static int local_LZ4_compress_fast1(const char* in, char* out, int inSize) { - return LZ4_compress_forceExtDict(&LZ4_dict, in, out, inSize); + return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 1); } +static int local_LZ4_compress_fast2(const char* in, char* out, int inSize) +{ + return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 2); +} -static void* stateLZ4HC; -static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) +static int local_LZ4_compress_fast17(const char* in, char* out, int inSize) { - return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize); + return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 17); } -static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) +static int local_LZ4_compress_fast_extState0(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0); } -static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) +static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0); } -static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) +static int local_LZ4_compress_withState(const char* in, char* out, int inSize) { - return LZ4_compressHC_continue((LZ4_streamHC_t*)ctx, in, out, inSize); + return LZ4_compress_withState(&LZ4_stream, in, out, inSize); } -static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) +static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compress_limitedOutput_withState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1); } -static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) +static int local_LZ4_compress_continue(const char* in, char* out, int inSize) { - return (int)LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL); + return LZ4_compress_continue(&LZ4_stream, in, out, inSize); } -static int local_LZ4_saveDict(const char* in, char* out, int inSize) +static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize) { - (void)in; - return LZ4_saveDict(&LZ4_dict, out, inSize); + return LZ4_compress_limitedOutput_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1); +} + +/* declare hidden function */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); + +static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize) +{ + return LZ4_compress_forceExtDict(&LZ4_stream, in, out, inSize); +} + + +/* HC compression functions */ +LZ4_streamHC_t LZ4_streamHC; +static void local_LZ4_resetStreamHC(void) +{ + LZ4_resetStreamHC(&LZ4_streamHC, 0); } -LZ4_streamHC_t LZ4_dictHC; static int local_LZ4_saveDictHC(const char* in, char* out, int inSize) { (void)in; - return LZ4_saveDictHC(&LZ4_dictHC, out, inSize); + return LZ4_saveDictHC(&LZ4_streamHC, out, inSize); } +static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_withStateHC(&LZ4_streamHC, in, out, inSize); +} -static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) +static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) { - (void)inSize; - //lz4_decode_sse((BYTE*)out, (BYTE*)in, inSize); - LZ4_decompress_fast(in, out, outSize); - return outSize; + return LZ4_compressHC_limitedOutput_withStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1); +} + +static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1); +} + +static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_continue(&LZ4_streamHC, in, out, inSize); } -static int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize) +static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_limitedOutput_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1); +} + + +/* decompression functions */ +static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) { (void)inSize; - LZ4_decompress_fast_withPrefix64k(in, out, outSize); + LZ4_decompress_fast(in, out, outSize); return outSize; } @@ -512,6 +542,13 @@ static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSi return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize); } + +/* frame functions */ +static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) +{ + return (int)LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL); +} + static LZ4F_decompressionContext_t g_dCtx; static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outSize) @@ -526,83 +563,65 @@ static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outS } +#define NB_COMPRESSION_ALGORITHMS 100 +#define NB_DECOMPRESSION_ALGORITHMS 100 int fullSpeedBench(char** fileNamesTable, int nbFiles) { int fileIdx=0; - char* orig_buff; -# define NB_COMPRESSION_ALGORITHMS 16 - double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0}; - double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0}; -# define NB_DECOMPRESSION_ALGORITHMS 9 - double totalDTime[NB_DECOMPRESSION_ALGORITHMS+1] = {0}; size_t errorCode; + /* Init */ errorCode = LZ4F_createDecompressionContext(&g_dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) - { - DISPLAY("dctx allocation issue \n"); - return 10; - } + if (LZ4F_isError(errorCode)) { DISPLAY("dctx allocation issue \n"); return 10; } - // Loop for each file + /* Loop for each fileName */ while (fileIdx<nbFiles) { FILE* inFile; + char* orig_buff = NULL; + struct chunkParameters* chunkP = NULL; + char* compressed_buff=NULL; char* inFileName; U64 inFileSize; size_t benchedSize; int nbChunks; int maxCompressedChunkSize; - struct chunkParameters* chunkP; size_t readSize; - char* compressed_buff; int compressedBuffSize; + int compressedBuffSize; U32 crcOriginal; - - // Init - stateLZ4 = LZ4_createStream(); - stateLZ4HC = LZ4_createStreamHC(); - - // Check file existence + /* Check file existence */ inFileName = fileNamesTable[fileIdx++]; inFile = fopen( inFileName, "rb" ); - if (inFile==NULL) - { - DISPLAY( "Pb opening %s\n", inFileName); - return 11; - } + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - // Memory allocation & restrictions + /* Memory size adjustments */ inFileSize = BMK_GetFileSize(inFileName); - if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; } - benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2; - if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; } + if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; } + benchedSize = (size_t) BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */ + if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; } if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) - { DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - } - // Alloc - chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)chunkSize)+1) * sizeof(struct chunkParameters)); - orig_buff = (char*) malloc((size_t)benchedSize); - nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize); - maxCompressedChunkSize = LZ4_compressBound(chunkSize); + /* Allocation */ + chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)g_chunkSize)+1) * sizeof(struct chunkParameters)); + orig_buff = (char*) malloc(benchedSize); + nbChunks = (int) ((benchedSize + (g_chunkSize-1)) / g_chunkSize); + maxCompressedChunkSize = LZ4_compressBound(g_chunkSize); compressedBuffSize = nbChunks * maxCompressedChunkSize; compressed_buff = (char*)malloc((size_t)compressedBuffSize); - - - if(!orig_buff || !compressed_buff) + if(!chunkP || !orig_buff || !compressed_buff) { - DISPLAY("\nError: not enough memory!\n"); - free(orig_buff); - free(compressed_buff); - free(chunkP); - fclose(inFile); - return 12; + DISPLAY("\nError: not enough memory!\n"); + fclose(inFile); + free(orig_buff); + free(compressed_buff); + free(chunkP); + return(12); } - // Fill input buffer + /* Fill in src buffer */ DISPLAY("Loading %s... \r", inFileName); readSize = fread(orig_buff, 1, benchedSize, inFile); fclose(inFile); @@ -616,11 +635,11 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) return 13; } - // Calculating input Checksum - crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0); + /* Calculating input Checksum */ + crcOriginal = XXH32(orig_buff, benchedSize,0); - // Bench + /* Bench */ { int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb; size_t cSize=0; @@ -629,67 +648,80 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) DISPLAY("\r%79s\r", ""); DISPLAY(" %s : \n", inFileName); - // Compression Algorithms - for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++) + /* Bench Compression Algorithms */ + for (cAlgNb=0; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (g_compressionTest); cAlgNb++) { const char* compressorName; int (*compressionFunction)(const char*, char*, int); - void* (*initFunction)(const char*) = NULL; + void (*initFunction)(void) = NULL; double bestTime = 100000000.; - // Init data chunks + /* filter compressionAlgo only */ + if ((g_compressionAlgo != ALL_COMPRESSORS) && (g_compressionAlgo != cAlgNb)) continue; + + /* Init data chunks */ { int i; size_t remaining = benchedSize; char* in = orig_buff; char* out = compressed_buff; - nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize); + nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); for (i=0; i<nbChunks; i++) { chunkP[i].id = i; - chunkP[i].origBuffer = in; in += chunkSize; - if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].origBuffer = in; in += g_chunkSize; + if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; chunkP[i].compressedSize = 0; } } - if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue; - switch(cAlgNb) { - case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break; - case 2 : compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break; - case 3 : compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break; - case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break; - case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break; - case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break; - case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break; - case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break; - case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break; - case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break; - case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break; - case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break; - case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break; - case 14: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; + case 0 : DISPLAY("Compression functions : \n"); continue; + case 1 : compressionFunction = local_LZ4_compress_default_large; compressorName = "LZ4_compress_default"; break; + case 2 : compressionFunction = local_LZ4_compress_default_small; compressorName = "LZ4_compress_default(small dst)"; break; + case 3 : compressionFunction = local_LZ4_compress_fast0; compressorName = "LZ4_compress_fast(0)"; break; + case 4 : compressionFunction = local_LZ4_compress_fast1; compressorName = "LZ4_compress_fast(1)"; break; + case 5 : compressionFunction = local_LZ4_compress_fast2; compressorName = "LZ4_compress_fast(2)"; break; + case 6 : compressionFunction = local_LZ4_compress_fast17; compressorName = "LZ4_compress_fast(17)"; break; + case 7 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break; + case 8 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break; + + case 10: compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break; + case 11: compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break; + case 12 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break; + case 13: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break; + case 14: compressionFunction = local_LZ4_compressHC_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_continue"; break; + case 15: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break; + case 20: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break; + case 30: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; chunkP[0].origSize = (int)benchedSize; nbChunks=1; break; - case 15: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict"; - LZ4_loadDict(&LZ4_dict, chunkP[0].origBuffer, chunkP[0].origSize); + case 40: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict"; + LZ4_loadDict(&LZ4_stream, chunkP[0].origBuffer, chunkP[0].origSize); break; - case 16: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC"; - LZ4_loadDictHC(&LZ4_dictHC, chunkP[0].origBuffer, chunkP[0].origSize); + case 41: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC"; + LZ4_loadDictHC(&LZ4_streamHC, chunkP[0].origBuffer, chunkP[0].origSize); break; - default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; + case 60: DISPLAY("Obsolete compression functions : \n"); continue; + case 61: compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break; + case 62: compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break; + case 63: compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break; + case 64: compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break; + case 65: compressionFunction = local_LZ4_compress_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_continue"; break; + case 66: compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_limitedOutput_continue"; break; + default : + continue; /* unknown ID : just skip */ } - for (loopNb = 1; loopNb <= nbIterations; loopNb++) + for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { double averageTime; int milliTime; PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize); - { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warming up memory + { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } /* warming up memory */ nb_loops = 0; milliTime = BMK_GetMilliStart(); @@ -697,17 +729,17 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { - if (initFunction!=NULL) ctx = (LZ4_stream_t*)initFunction(chunkP[0].origBuffer); + if (initFunction!=NULL) initFunction(); for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1); } - if (initFunction!=NULL) free(ctx); nb_loops++; } milliTime = BMK_GetMilliSpan(milliTime); + nb_loops += !nb_loops; /* avoid division by zero */ averageTime = (double)milliTime / nb_loops; if (averageTime < bestTime) bestTime = averageTime; cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; @@ -719,24 +751,22 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); else DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); - - totalCTime[cAlgNb] += bestTime; - totalCSize[cAlgNb] += cSize; } - // Prepare layout for decompression - // Init data chunks + /* Prepare layout for decompression */ + /* Init data chunks */ { int i; size_t remaining = benchedSize; char* in = orig_buff; char* out = compressed_buff; - nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize); + + nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); for (i=0; i<nbChunks; i++) { chunkP[i].id = i; - chunkP[i].origBuffer = in; in += chunkSize; - if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].origBuffer = in; in += g_chunkSize; + if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; chunkP[i].compressedSize = 0; } @@ -747,39 +777,45 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1); } - // Decompression Algorithms - for (dAlgNb=1; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); dAlgNb++) + /* Decompression Algorithms */ + for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (g_decompressionTest); dAlgNb++) { - //const char* dName = decompressionNames[dAlgNb]; const char* dName; int (*decompressionFunction)(const char*, char*, int, int); double bestTime = 100000000.; - if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb)) continue; + if ((g_decompressionAlgo != ALL_DECOMPRESSORS) && (g_decompressionAlgo != dAlgNb)) continue; switch(dAlgNb) { + case 0: DISPLAY("Decompression functions : \n"); continue; case 1: decompressionFunction = local_LZ4_decompress_fast; dName = "LZ4_decompress_fast"; break; - case 2: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; dName = "LZ4_decompress_fast_withPrefix64k"; break; case 3: decompressionFunction = local_LZ4_decompress_fast_usingDict; dName = "LZ4_decompress_fast_usingDict"; break; case 4: decompressionFunction = LZ4_decompress_safe; dName = "LZ4_decompress_safe"; break; - case 5: decompressionFunction = LZ4_decompress_safe_withPrefix64k; dName = "LZ4_decompress_safe_withPrefix64k"; break; case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break; case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; break; case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break; case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); - if (LZ4F_isError(errorCode)) { DISPLAY("Preparation error compressing frame\n"); return 1; } + if (LZ4F_isError(errorCode)) + { + DISPLAY("Error while preparing compressed frame\n"); + free(orig_buff); + free(compressed_buff); + free(chunkP); + return 1; + } chunkP[0].origSize = (int)benchedSize; chunkP[0].compressedSize = (int)errorCode; nbChunks = 1; break; - default : DISPLAY("ERROR ! Bad decompression algorithm Id !! \n"); free(chunkP); return 1; + default : + continue; /* skip if unknown ID */ } - { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking + { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing source area, for CRC checking */ - for (loopNb = 1; loopNb <= nbIterations; loopNb++) + for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { double averageTime; int milliTime; @@ -802,6 +838,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) } milliTime = BMK_GetMilliSpan(milliTime); + nb_loops += !nb_loops; /* Avoid division by zero */ averageTime = (double)milliTime / nb_loops; if (averageTime < bestTime) bestTime = averageTime; @@ -813,24 +850,21 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) } DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); - - totalDTime[dAlgNb] += bestTime; } - } - free(orig_buff); free(compressed_buff); free(chunkP); } - if (BMK_pause) { printf("press enter...\n"); getchar(); } + LZ4F_freeDecompressionContext(g_dCtx); + if (g_pause) { printf("press enter...\n"); (void)getchar(); } return 0; } -int usage(char* exename) +static int usage(char* exename) { DISPLAY( "Usage :\n"); DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); @@ -841,7 +875,7 @@ int usage(char* exename) return 0; } -int usage_advanced(void) +static int usage_advanced(void) { DISPLAY( "\nAdvanced options :\n"); DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS); @@ -851,7 +885,7 @@ int usage_advanced(void) return 0; } -int badusage(char* exename) +static int badusage(char* exename) { DISPLAY("Wrong parameters\n"); usage(exename); @@ -877,7 +911,7 @@ int main(int argc, char** argv) if(!argument) continue; // Protection if argument empty if (!strcmp(argument, "--no-prompt")) { - no_prompt = 1; + g_noPrompt = 1; continue; } @@ -892,22 +926,22 @@ int main(int argc, char** argv) { // Select compression algorithm only case 'c': - decompressionTest = 0; + g_decompressionTest = 0; while ((argument[1]>= '0') && (argument[1]<= '9')) { - compressionAlgo *= 10; - compressionAlgo += argument[1] - '0'; + g_compressionAlgo *= 10; + g_compressionAlgo += argument[1] - '0'; argument++; } break; // Select decompression algorithm only case 'd': - compressionTest = 0; + g_compressionTest = 0; while ((argument[1]>= '0') && (argument[1]<= '9')) { - decompressionAlgo *= 10; - decompressionAlgo += argument[1] - '0'; + g_decompressionAlgo *= 10; + g_decompressionAlgo += argument[1] - '0'; argument++; } break; @@ -928,7 +962,7 @@ int main(int argc, char** argv) { int B = argument[1] - '0'; int S = 1 << (8 + 2*B); - BMK_SetBlocksize(S); + BMK_setBlocksize(S); argument++; break; } @@ -940,16 +974,16 @@ _exit_blockProperties: // Modify Nb Iterations case 'i': - if ((argument[1] >='1') && (argument[1] <='9')) + if ((argument[1] >='0') && (argument[1] <='9')) { int iters = argument[1] - '0'; - BMK_SetNbIterations(iters); + BMK_setNbIterations(iters); argument++; } break; // Pause at the end (hidden option) - case 'p': BMK_SetPause(); break; + case 'p': BMK_setPause(); break; // Unknown command default : badusage(exename); return 1; diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 3d3cf8e..6588290 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -19,7 +19,6 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4 - LZ4 source mirror : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ @@ -154,10 +153,10 @@ static U32 FUZ_rand(U32* src) { U32 rand32 = *src; rand32 *= PRIME1; - rand32 += PRIME2; + rand32 ^= PRIME2; rand32 = FUZ_rotl32(rand32, 13); *src = rand32; - return rand32 >> 3; + return rand32; } @@ -169,27 +168,28 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou size_t pos = 0; U32 P32 = (U32)(32768 * proba); - // First Byte - BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); + /* First Bytes */ + while (pos < 20) + BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); while (pos < bufferSize) { - // Select : Literal (noise) or copy (within 64K) + /* Select : Literal (noise) or copy (within 64K) */ if (FUZ_RAND15BITS < P32) { - // Copy (within 64K) + /* Copy (within 64K) */ size_t match, d; size_t length = FUZ_RANDLENGTH + 4; size_t offset = FUZ_RAND15BITS + 1; - if (offset > pos) offset = pos; + while (offset > pos) offset >>= 1; d = pos + length; - if (d > bufferSize) d = bufferSize; + while (d > bufferSize) d = bufferSize; match = pos - offset; while (pos < d) BBuffer[pos++] = BBuffer[match++]; } else { - // Literal (noise) + /* Literal (noise) */ size_t d; size_t length = FUZ_RANDLENGTH; d = pos + length; @@ -210,7 +210,7 @@ static int FUZ_AddressOverflow(void) printf("Overflow tests : "); - // Only possible in 32-bits + /* Only possible in 32-bits */ if (sizeof(void*)==8) { printf("64 bits mode : no overflow \n"); @@ -300,7 +300,17 @@ static void FUZ_displayUpdate(unsigned testNb) } -static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const double compressibility) +static void FUZ_findDiff(const void* buff1, const void* buff2) +{ + const BYTE* b1 = (const BYTE*)buff1; + const BYTE* b2 = (const BYTE*)buff2; + size_t i=0; + while (b1[i]==b2[i]) i++; + DISPLAY("Wrong Byte at position %u\n", (unsigned)i); +} + + +static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration) { unsigned long long bytes = 0; unsigned long long cbytes = 0; @@ -313,34 +323,35 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do int ret; unsigned cycleNb; # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } + printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } # define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); } void* stateLZ4 = malloc(LZ4_sizeofState()); void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); - void* LZ4continue; LZ4_stream_t LZ4dict; LZ4_streamHC_t LZ4dictHC; U32 crcOrig, crcCheck; U32 coreRandState = seed; U32 randState = coreRandState ^ PRIME3; int result = 0; + const U32 startTime = FUZ_GetMilliStart(); - // init + /* init */ memset(&LZ4dict, 0, sizeof(LZ4dict)); + duration *= 1000; - // Create compressible test buffer + /* Create compressible test buffer */ CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); compressedBuffer = (char*)malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); - // move to startCycle + /* move to startCycle */ for (cycleNb = 0; cycleNb < startCycle; cycleNb++) { (void)FUZ_rand(&coreRandState); - if (0) // some problems related to dictionary re-use; in this case, enable this loop + if (0) /* some problems are related to dictionary re-use; in this case, enable this loop */ { int dictSize, blockSize, blockStart; char* dict; @@ -362,8 +373,8 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do } } - // Test loop - for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++) + /* Main test loop */ + for (cycleNb = startCycle; (cycleNb < nbCycles) || (FUZ_GetMilliSpan(startTime) < duration) ; cycleNb++) { U32 testNb = 0; char* dict; @@ -375,7 +386,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do (void)FUZ_rand(&coreRandState); randState = coreRandState ^ PRIME3; - // Select block to test + /* Select block to test */ blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; @@ -385,23 +396,58 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do /* Compression tests */ - // Test compression HC + /* Test compression destSize */ + FUZ_DISPLAYTEST; + { + int srcSize = blockSize; + int targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; + char endCheck = FUZ_rand(&randState) & 255; + compressedBuffer[targetSize] = endCheck; + ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize); + FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !"); + FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !"); + FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !"); + DISPLAY("destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); + if (targetSize>0) + { + FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); + /* check correctness */ + FUZ_DISPLAYTEST; + + crcOrig = XXH32(block, srcSize, 0); + compressedSize = ret; + endCheck = FUZ_rand(&randState) & 255; + decodedBuffer[srcSize] = endCheck; + ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize"); + FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + FUZ_CHECKTEST(decodedBuffer[srcSize] != endCheck, "LZ4_decompress_safe() overwrite dst buffer !"); + crcCheck = XXH32(decodedBuffer, srcSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe() corrupted decoded data"); + + DISPLAY(" OK \n"); + } + else + DISPLAY(" \n"); + } + + /* Test compression HC */ FUZ_DISPLAYTEST; ret = LZ4_compressHC(block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); HCcompressedSize = ret; - // Test compression HC using external state + /* Test compression HC using external state */ FUZ_DISPLAYTEST; ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); - // Test compression using external state + /* Test compression using external state */ FUZ_DISPLAYTEST; ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); - // Test compression + /* Test compression */ FUZ_DISPLAYTEST; ret = LZ4_compress(block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compress() failed"); @@ -411,7 +457,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do crcOrig = XXH32(block, blockSize, 0); - // Test decoding with output size being exactly what's necessary => must work + /* Test decoding with output size being exactly what's necessary => must work */ FUZ_DISPLAYTEST; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); @@ -419,19 +465,19 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do crcCheck = XXH32(decodedBuffer, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); - // Test decoding with one byte missing => must fail + /* Test decoding with one byte missing => must fail */ FUZ_DISPLAYTEST; decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer"); - // Test decoding with one byte too much => must fail + /* Test decoding with one byte too much => must fail */ FUZ_DISPLAYTEST; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); - // Test decoding with output size exactly what's necessary => must work + /* Test decoding with output size exactly what's necessary => must work */ FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); @@ -545,16 +591,18 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do /* Compress using dictionary */ FUZ_DISPLAYTEST; - LZ4continue = LZ4_create (dict); - LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables - blockContinueCompressedSize = LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); - free (LZ4continue); + { + LZ4_stream_t LZ4_stream; + LZ4_resetStream(&LZ4_stream); + LZ4_compress_continue (&LZ4_stream, dict, compressedBuffer, dictSize); /* Just to fill hash tables */ + blockContinueCompressedSize = LZ4_compress_continue (&LZ4_stream, block, compressedBuffer, blockSize); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); + } /* Decompress with dictionary as prefix */ FUZ_DISPLAYTEST; memcpy(decodedBuffer, dict, dictSize); - ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize); + ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input"); crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); if (crcCheck!=crcOrig) @@ -567,14 +615,14 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize); FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data"); + ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); + FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data"); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - // Compress using External dictionary + /* Compress using External dictionary */ FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 0xF) + 1; // Separation, so it is an ExtDict + dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_loadDict(&LZ4dict, dict, dictSize); blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); @@ -583,7 +631,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_DISPLAYTEST; LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); + FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); FUZ_DISPLAYTEST; LZ4_loadDict(&LZ4dict, dict, dictSize); @@ -591,7 +639,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); - // Decompress with dictionary as external + /* Decompress with dictionary as external */ FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); @@ -599,11 +647,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size") crcCheck = XXH32(decodedBuffer, blockSize, 0); if (crcCheck!=crcOrig) - { - int i=0; - while (block[i]==decodedBuffer[i]) i++; - printf("Wrong Byte at position %i/%i\n", i, blockSize); - } + FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); FUZ_DISPLAYTEST; @@ -617,7 +661,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_DISPLAYTEST; decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)"); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); FUZ_DISPLAYTEST; @@ -638,9 +682,9 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do } } - // Compress HC using External dictionary + /* Compress HC using External dictionary */ FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 7); // even bigger separation + dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); @@ -665,29 +709,26 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") crcCheck = XXH32(decodedBuffer, blockSize, 0); if (crcCheck!=crcOrig) - { - int i=0; - while (block[i]==decodedBuffer[i]) i++; - printf("Wrong Byte at position %i/%i\n", i, blockSize); - } + FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - - // ***** End of tests *** // - // Fill stats + /* ***** End of tests *** */ + /* Fill stats */ bytes += blockSize; cbytes += compressedSize; hcbytes += HCcompressedSize; ccbytes += blockContinueCompressedSize; } + if (nbCycles<=1) nbCycles = cycleNb; /* end by time */ + bytes += !bytes; /* avoid division by 0 */ printf("\r%7u /%7u - ", cycleNb, nbCycles); printf("all tests completed successfully \n"); printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100); - // unalloc + /* release memory */ { _exit: free(CNBuffer); @@ -719,13 +760,13 @@ static void FUZ_unitTests(void) char ringBuffer[ringBufferSize]; U32 randState = 1; - // Init + /* Init */ FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState); - // 32-bits address space overflow test + /* 32-bits address space overflow test */ FUZ_AddressOverflow(); - // LZ4 streaming tests + /* LZ4 streaming tests */ { LZ4_stream_t* statePtr; LZ4_stream_t streamingState; @@ -733,12 +774,12 @@ static void FUZ_unitTests(void) U64 crcNew; int result; - // Allocation test + /* Allocation test */ statePtr = LZ4_createStream(); FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed"); LZ4_freeStream(statePtr); - // simple compression test + /* simple compression test */ crcOrig = XXH64(testInput, testCompressedSize, 0); LZ4_resetStream(&streamingState); result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1); @@ -749,7 +790,7 @@ static void FUZ_unitTests(void) crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - // ring buffer test + /* ring buffer test */ { XXH64_state_t xxhOrig; XXH64_state_t xxhNew; @@ -783,7 +824,7 @@ static void FUZ_unitTests(void) crcNew = XXH64_digest(&xxhNew); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - // prepare next message + /* prepare next message */ iNext += messageSize; rNext += messageSize; dNext += messageSize; @@ -794,21 +835,20 @@ static void FUZ_unitTests(void) } } - // LZ4 HC streaming tests + /* LZ4 HC streaming tests */ { LZ4_streamHC_t* sp; LZ4_streamHC_t sHC; - //XXH64_state_t xxh; U64 crcOrig; U64 crcNew; int result; - // Allocation test + /* Allocation test */ sp = LZ4_createStreamHC(); FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); LZ4_freeStreamHC(sp); - // simple compression test + /* simple HC compression test */ crcOrig = XXH64(testInput, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); @@ -819,7 +859,7 @@ static void FUZ_unitTests(void) crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - // simple dictionary compression test + /* simple dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); LZ4_loadDictHC(&sHC, testInput, 64 KB); @@ -831,7 +871,7 @@ static void FUZ_unitTests(void) crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); - // multiple HC compression test with dictionary + /* multiple HC compression test with dictionary */ { int result1, result2; int segSize = testCompressedSize / 2; @@ -851,7 +891,7 @@ static void FUZ_unitTests(void) FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } - // remote dictionary HC compression test + /* remote dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); LZ4_loadDictHC(&sHC, testInput, 32 KB); @@ -863,7 +903,7 @@ static void FUZ_unitTests(void) crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); - // multiple HC compression with ext. dictionary + /* multiple HC compression with ext. dictionary */ { XXH64_state_t crcOrigState; XXH64_state_t crcNewState; @@ -912,7 +952,7 @@ static void FUZ_unitTests(void) } } - // ring buffer test + /* ring buffer test */ { XXH64_state_t xxhOrig; XXH64_state_t xxhNew; @@ -956,24 +996,51 @@ static void FUZ_unitTests(void) } } - // small decoder-side ring buffer test + /* small decoder-side ring buffer test */ { XXH64_state_t xxhOrig; XXH64_state_t xxhNew; LZ4_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 10; + const U32 maxMessageSizeLog = 12; const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1; - U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; + U32 messageSize; U32 totalMessageSize = 0; U32 iNext = 0; U32 dNext = 0; - const U32 dBufferSize = 64 KB + maxMessageSizeMask; + const U32 dBufferSize = 64 KB; XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNew, 0); LZ4_resetStreamHC(&sHC, 0); LZ4_setStreamDecode(&decodeState, NULL, 0); +#define BSIZE1 65537 +#define BSIZE2 16435 + + /* first block */ + + messageSize = BSIZE1; + XXH64_update(&xxhOrig, testInput + iNext, messageSize); + crcOrig = XXH64_digest(&xxhOrig); + + result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); + + result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); + FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); + + XXH64_update(&xxhNew, testVerify + dNext, messageSize); + crcNew = XXH64_digest(&xxhNew); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + + /* prepare next message */ + dNext += messageSize; + totalMessageSize += messageSize; + messageSize = BSIZE2; + iNext = 132000; + memcpy(testInput + iNext, testInput + 8, messageSize); + if (dNext > dBufferSize) dNext = 0; + while (totalMessageSize < 9 MB) { XXH64_update(&xxhOrig, testInput + iNext, messageSize); @@ -983,18 +1050,20 @@ static void FUZ_unitTests(void) FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed"); + FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); XXH64_update(&xxhNew, testVerify + dNext, messageSize); crcNew = XXH64_digest(&xxhNew); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + if (crcOrig != crcNew) + FUZ_findDiff(testInput + iNext, testVerify + dNext); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test"); /* prepare next message */ dNext += messageSize; totalMessageSize += messageSize; messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; iNext = (FUZ_rand(&randState) & 65535); - if (dNext + messageSize > dBufferSize) dNext = 0; + if (dNext > dBufferSize) dNext = 0; } } } @@ -1013,6 +1082,7 @@ static int FUZ_usage(char* programName) DISPLAY( "\n"); DISPLAY( "Arguments :\n"); DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS); + DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); DISPLAY( " -s# : Select seed (default:prompt user)\n"); DISPLAY( " -t# : Select starting test number (default:0)\n"); DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); @@ -1033,8 +1103,9 @@ int main(int argc, char** argv) int proba = FUZ_COMPRESSIBILITY_DEFAULT; int pause = 0; char* programName = argv[0]; + U32 duration = 0; - // Check command line + /* Check command line */ for(argNb=1; argNb<argc; argNb++) { char* argument = argv[argNb]; @@ -1066,7 +1137,7 @@ int main(int argc, char** argv) case 'i': argument++; - nbTests=0; + nbTests = 0; duration = 0; while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; @@ -1075,6 +1146,31 @@ int main(int argc, char** argv) } break; + case 'T': + argument++; + nbTests = 0; duration = 0; + for (;;) + { + switch(*argument) + { + case 'm': duration *= 60; argument++; continue; + case 's': + case 'n': argument++; continue; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': duration *= 10; duration += *argument++ - '0'; continue; + } + break; + } + break; + case 's': argument++; seed=0; seedset=1; @@ -1115,7 +1211,6 @@ int main(int argc, char** argv) } } - // Get Seed printf("Starting LZ4 fuzzer (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION); if (!seedset) seed = FUZ_GetMilliStart() % 10000; @@ -1127,11 +1222,11 @@ int main(int argc, char** argv) if (nbTests<=0) nbTests=1; { - int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100); + int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); if (pause) { DISPLAY("press enter ... \n"); - getchar(); + (void)getchar(); } return result; } diff --git a/programs/lz4.1 b/programs/lz4.1 index 1225832..a79fed3 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -158,13 +158,13 @@ for files that have not been compressed with force write to standard output, even if it is the console .TP -.BR \-m +.BR \-m ", " \--multiple Multiple file names. By default, the second filename is used as the output filename for the compressed file. With .B -m -, you can specify any number of input filenames, each of them will be compressed -with the resulting compressed file named +, you can specify any number of input filenames. Each of them will be compressed +independently, and the resulting name of the compressed file will be .B filename.lz4 . @@ -177,15 +177,15 @@ with the resulting compressed file named block dependency (improve compression ratio) .TP .B \--[no-]frame-crc - disable stream checksum (default:enabled) + select frame checksum (default:enabled) .TP .B \--[no-]content-size - compressed file includes original size (default:not present) + header includes original size (default:not present) Note : this option can only be activated when the original size can be determined, -hence for a file. It won't work with unknown source size, such as stdin pipe. +hence for a file. It won't work with unknown source size, such as stdin or pipe. .TP .B \--[no-]sparse - enable sparse file (default:disabled)(experimental) + sparse file support (default:enabled) .TP .B \-l use Legacy format (useful for Linux Kernel compression) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index da5da71..970856d 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -65,7 +65,7 @@ /**************************** * OS-specific Includes *****************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) # include <io.h> /* _isatty */ # ifdef __MINGW32__ int _fileno(FILE *stream); /* MINGW somehow forgets to include this prototype into <stdio.h> */ @@ -102,7 +102,7 @@ ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned displayLevel = 2; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ +static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */ /************************************** @@ -175,7 +175,7 @@ static int usage_advanced(void) /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */ DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n"); DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); - DISPLAY( "--sparse : enable sparse file (default:disabled)(experimental)\n"); + DISPLAY( "--[no-]sparse : sparse file support (default:enabled)\n"); DISPLAY( "Benchmark arguments :\n"); DISPLAY( " -b : benchmark file(s)\n"); DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); @@ -251,7 +251,7 @@ static int badusage(void) static void waitEnter(void) { DISPLAY("Press enter to continue...\n"); - getchar(); + (void)getchar(); } @@ -265,7 +265,8 @@ int main(int argc, char** argv) forceStdout=0, forceCompress=0, main_pause=0, - multiple_inputs=0; + multiple_inputs=0, + operationResult=0; const char* input_filename=0; const char* output_filename=0; char* dynNameSpace=0; @@ -294,12 +295,13 @@ int main(int argc, char** argv) /* long commands (--long-word) */ if (!strcmp(argument, "--compress")) { forceCompress = 1; continue; } if ((!strcmp(argument, "--decompress")) - || (!strcmp(argument, "--uncompress"))) { decode = 1; continue; } + || (!strcmp(argument, "--uncompress"))) { decode = 1; continue; } + if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; if (inFileNames==NULL) inFileNames = (const char**)malloc(argc * sizeof(char*)); continue; } if (!strcmp(argument, "--test")) { decode = 1; LZ4IO_setOverwrite(1); output_filename=nulmark; continue; } if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; } if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; } if ((!strcmp(argument, "--stdout")) - || (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; } + || (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; } if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; } if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; } if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; } @@ -311,6 +313,7 @@ int main(int argc, char** argv) if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; } if (!strcmp(argument, "--keep")) { continue; } /* keep source file (default anyway; just for xz/lzma compatibility) */ + /* Short commands (note : aggregated short commands are allowed) */ if (argument[0]=='-') { @@ -420,11 +423,15 @@ int main(int argc, char** argv) /* Modify Nb Iterations (benchmark only) */ case 'i': - if ((argument[1] >='1') && (argument[1] <='9')) { - int iters = argument[1] - '0'; + unsigned iters = 0; + while ((argument[1] >='0') && (argument[1] <='9')) + { + iters *= 10; + iters += argument[1] - '0'; + argument++; + } BMK_setNbIterations(iters); - argument++; } break; @@ -463,14 +470,19 @@ int main(int argc, char** argv) if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10); /* No input filename ==> use stdin */ - if (multiple_inputs) input_filename = inFileNames[0], output_filename = (char*)(inFileNames[0]); + if (multiple_inputs) input_filename = inFileNames[0], output_filename = (const char*)(inFileNames[0]); if(!input_filename) { input_filename=stdinmark; } /* Check if input or output are defined as console; trigger an error in this case */ if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); /* Check if benchmark is selected */ - if (bench) return BMK_benchFiles(inFileNames, ifnIdx, cLevel); + if (bench) + { + int bmkResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel); + free((void*)inFileNames); + return bmkResult; + } /* No output filename ==> try to select one automatically (when possible) */ while (!output_filename) @@ -509,32 +521,40 @@ int main(int argc, char** argv) /* Check if output is defined as console; trigger an error in this case */ if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage(); - /* No warning message in pure pipe mode (stdin + stdout) */ + /* Downgrade notification level in pure pipe mode (stdin + stdout) and multiple file mode */ if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; + if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; /* IO Stream/File */ LZ4IO_setNotificationLevel(displayLevel); - if (decode) DEFAULT_DECOMPRESSOR(input_filename, output_filename); + if (decode) + { + if (multiple_inputs) + operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION); + else + DEFAULT_DECOMPRESSOR(input_filename, output_filename); + } else { - /* compression is default action */ - if (legacy_format) - { - DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n"); - LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); - } + /* compression is default action */ + if (legacy_format) + { + DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n"); + LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); + } + else + { + if (multiple_inputs) + operationResult = LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel); else - { - if (multiple_inputs) - LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel); - else - DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); - } + DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); + } } if (main_pause) waitEnter(); free(dynNameSpace); free((void*)inFileNames); + if (operationResult != 0) return operationResult; return 0; } diff --git a/programs/lz4io.c b/programs/lz4io.c index 02e03c8..209f5ed 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -46,22 +46,22 @@ /***************************** * Includes *****************************/ -#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */ +#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include <stdlib.h> /* malloc, free */ #include <string.h> /* strcmp, strlen */ #include <time.h> /* clock */ #include <sys/types.h> /* stat64 */ #include <sys/stat.h> /* stat64 */ #include "lz4io.h" -#include "lz4.h" /* still required for legacy format */ -#include "lz4hc.h" /* still required for legacy format */ +#include "lz4.h" /* still required for legacy format */ +#include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" /****************************** * OS-specific Includes ******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) # include <fcntl.h> /* _O_BINARY */ # include <io.h> /* _setmode, _fileno, _get_osfhandle */ # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) @@ -131,7 +131,7 @@ static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; static int g_blockChecksum = 0; static int g_streamChecksum = 1; static int g_blockIndependence = 1; -static int g_sparseFileSupport = 0; +static int g_sparseFileSupport = 1; static int g_contentSizeFlag = 0; static const int minBlockSizeID = 4; @@ -159,7 +159,7 @@ static const int maxBlockSizeID = 7; #define EXTENDED_ARGUMENTS #define EXTENDED_HELP #define EXTENDED_FORMAT -#define DEFAULT_DECOMPRESSOR decodeLZ4S +#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F /* ************************************************** */ @@ -240,7 +240,7 @@ static unsigned long long LZ4IO_GetFileSize(const char* infilename) struct stat statbuf; r = stat(infilename, &statbuf); #endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* failure, or is not a regular file */ return (unsigned long long)statbuf.st_size; } @@ -253,7 +253,7 @@ static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id)) static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; } -static int get_fileHandle(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput) +static int LZ4IO_getFiles(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput) { if (!strcmp (input_filename, stdinmark)) @@ -267,6 +267,12 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam *pfinput = fopen(input_filename, "rb"); } + if ( *pfinput==0 ) + { + DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", input_filename); + return 1; + } + if (!strcmp (output_filename, stdoutmark)) { DISPLAYLEVEL(4,"Using stdout for output\n"); @@ -283,18 +289,18 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam fclose(*pfoutput); if (!g_overwrite) { - char ch; + int ch = 'Y'; DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); - DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); - if (g_displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ - ch = (char)getchar(); - if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); + if ((g_displayLevel <= 1) || (*pfinput == stdin)) + EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ + DISPLAYLEVEL(2, "Overwrite ? (Y/n) : "); + while((ch = getchar()) != '\n' && ch != EOF) /* flush integrated */ + if ((ch!='Y') && (ch!='y')) EXM_THROW(12, "No. Operation aborted : %s already exists", output_filename); } } *pfoutput = fopen( output_filename, "wb" ); } - if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename); if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename); return 0; @@ -316,16 +322,23 @@ static void LZ4IO_writeLE32 (void* p, unsigned value32) dstPtr[3] = (unsigned char)(value32 >> 24); } +static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSize, int cLevel) +{ + (void)cLevel; + return LZ4_compress_fast(src, dst, srcSize, dstSize, 1); +} + /* LZ4IO_compressFilename_Legacy : * This function is intentionally "hidden" (not published in .h) * It generates compressed streams using the old 'legacy' format */ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel) { - int (*compressionFunction)(const char*, char*, int); + int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); unsigned long long filesize = 0; unsigned long long compressedfilesize = MAGICNUMBER_SIZE; char* in_buff; char* out_buff; + const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE); FILE* finput; FILE* foutput; clock_t start, end; @@ -334,14 +347,14 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output /* Init */ start = clock(); - if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; + if (compressionlevel < 3) compressionFunction = LZ4IO_LZ4_compress; else compressionFunction = LZ4_compress_HC; - get_fileHandle(input_filename, output_filename, &finput, &foutput); - if ((g_displayLevel==2) && (compressionlevel==1)) g_displayLevel=3; + if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput)) + EXM_THROW(20, "File error"); /* Allocate Memory */ in_buff = (char*)malloc(LEGACY_BLOCKSIZE); - out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); + out_buff = (char*)malloc(outBuffSize); if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); /* Write Archive Header */ @@ -359,9 +372,9 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output filesize += inSize; /* Compress Block */ - outSize = compressionFunction(in_buff, out_buff+4, inSize); + outSize = compressionFunction(in_buff, out_buff+4, inSize, outBuffSize, compressionlevel); compressedfilesize += outSize+4; - DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ LZ4IO_writeLE32(out_buff, outSize); @@ -372,6 +385,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output /* Status */ end = clock(); DISPLAYLEVEL(2, "\r%79s\r", ""); + filesize += !filesize; /* avoid divide by zero */ DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); { @@ -393,128 +407,210 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output * Compression using Frame format *********************************************/ -int LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionLevel) +typedef struct { + void* srcBuffer; + size_t srcBufferSize; + void* dstBuffer; + size_t dstBufferSize; + LZ4F_compressionContext_t ctx; +} cRess_t; + +static cRess_t LZ4IO_createCResources(void) +{ + const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); + cRess_t ress; + LZ4F_errorCode_t errorCode; + + errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION); + if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); + + /* Allocate Memory */ + ress.srcBuffer = malloc(blockSize); + ress.srcBufferSize = blockSize; + ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */ + ress.dstBuffer = malloc(ress.dstBufferSize); + if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); + + return ress; +} + +static void LZ4IO_freeCResources(cRess_t ress) +{ + LZ4F_errorCode_t errorCode; + free(ress.srcBuffer); + free(ress.dstBuffer); + errorCode = LZ4F_freeCompressionContext(ress.ctx); + if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); +} + +/* + * LZ4IO_compressFilename_extRess() + * result : 0 : compression completed correctly + * 1 : missing or pb opening srcFileName + */ +static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) { unsigned long long filesize = 0; unsigned long long compressedfilesize = 0; - char* in_buff; - char* out_buff; - FILE* finput; - FILE* foutput; - clock_t start, end; - int blockSize; - size_t sizeCheck, headerSize, readSize, outBuffSize; - LZ4F_compressionContext_t ctx; - LZ4F_errorCode_t errorCode; + FILE* srcFile; + FILE* dstFile; + void* const srcBuffer = ress.srcBuffer; + void* const dstBuffer = ress.dstBuffer; + const size_t dstBufferSize = ress.dstBufferSize; + const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); + size_t sizeCheck, headerSize, readSize; + LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */ LZ4F_preferences_t prefs; /* Init */ - start = clock(); memset(&prefs, 0, sizeof(prefs)); - if ((g_displayLevel==2) && (compressionLevel>=3)) g_displayLevel=3; - errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); - get_fileHandle(input_filename, output_filename, &finput, &foutput); - blockSize = LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); + + /* File check */ + if (LZ4IO_getFiles(srcFileName, dstFileName, &srcFile, &dstFile)) return 1; /* Set compression parameters */ prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence; - prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId; - prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum; + prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence; + prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId; + prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum; if (g_contentSizeFlag) { - unsigned long long fileSize = LZ4IO_GetFileSize(input_filename); + unsigned long long fileSize = LZ4IO_GetFileSize(srcFileName); prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ + if (fileSize==0) + DISPLAYLEVEL(3, "Warning : cannot determine uncompressed frame content size \n"); } - /* Allocate Memory */ - in_buff = (char*)malloc(blockSize); - outBuffSize = LZ4F_compressBound(blockSize, &prefs); - out_buff = (char*)malloc(outBuffSize); - if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); - - /* Write Archive Header */ - headerSize = LZ4F_compressBegin(ctx, out_buff, outBuffSize, &prefs); - if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); - sizeCheck = fwrite(out_buff, 1, headerSize, foutput); - if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); - compressedfilesize += headerSize; - /* read first block */ - readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); + readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile); filesize += readSize; - /* Main Loop */ - while (readSize>0) + /* single-block file */ + if (readSize < blockSize) { - size_t outSize; - - /* Compress Block */ - outSize = LZ4F_compressUpdate(ctx, out_buff, outBuffSize, in_buff, readSize, NULL); - if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); - compressedfilesize += outSize; - DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + /* Compress in single pass */ + size_t cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs); + if (LZ4F_isError(cSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(cSize)); + compressedfilesize += cSize; + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", + (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ /* Write Block */ - sizeCheck = fwrite(out_buff, 1, outSize, foutput); - if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); - - /* Read next block */ - readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); - filesize += readSize; + sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); + if (sizeCheck!=cSize) EXM_THROW(35, "Write error : cannot write compressed block"); } - /* End of Stream mark */ - headerSize = LZ4F_compressEnd(ctx, out_buff, outBuffSize, NULL); - if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); + else - sizeCheck = fwrite(out_buff, 1, headerSize, foutput); - if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream"); - compressedfilesize += headerSize; + /* multiple-blocks file */ + { + /* Write Archive Header */ + headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); + if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); + sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); + if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); + compressedfilesize += headerSize; + + /* Main Loop */ + while (readSize>0) + { + size_t outSize; - /* Close & Free */ - free(in_buff); - free(out_buff); - fclose(finput); - fclose(foutput); - errorCode = LZ4F_freeCompressionContext(ctx); - if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); + /* Compress Block */ + outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); + if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); + compressedfilesize += outSize; + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); + + /* Write Block */ + sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); + if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); + + /* Read next block */ + readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); + filesize += readSize; + } + + /* End of Stream mark */ + headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); + if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); + + sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); + if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream"); + compressedfilesize += headerSize; + } + + /* Release files */ + fclose (srcFile); + fclose (dstFile); /* Final Status */ - end = clock(); DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */ + + return 0; +} + + +int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel) +{ + clock_t start, end; + cRess_t ress; + int issueWithSrcFile = 0; + + /* Init */ + start = clock(); + ress = LZ4IO_createCResources(); + + /* Compress File */ + issueWithSrcFile += LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); + + /* Free resources */ + LZ4IO_freeCResources(ress); + + /* Final Status */ + end = clock(); { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + double seconds = (double)(end - start) / CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); } - return 0; + return issueWithSrcFile; } #define FNSPACE 30 -int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel) +int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) { int i; - char* outFileName = (char*)malloc(FNSPACE); + int missed_files = 0; + char* dstFileName = (char*)malloc(FNSPACE); size_t ofnSize = FNSPACE; const size_t suffixSize = strlen(suffix); + cRess_t ress; + + /* init */ + ress = LZ4IO_createCResources(); + /* loop on each file */ for (i=0; i<ifntSize; i++) { size_t ifnSize = strlen(inFileNamesTable[i]); - if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); } - strcpy(outFileName, inFileNamesTable[i]); - strcat(outFileName, suffix); - LZ4IO_compressFilename(inFileNamesTable[i], outFileName, compressionlevel); + if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); } + strcpy(dstFileName, inFileNamesTable[i]); + strcat(dstFileName, suffix); + + missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel); } - free(outFileName); - return 0; + + /* Close & Free */ + LZ4IO_freeCResources(ress); + free(dstFileName); + + return missed_files; } @@ -528,15 +624,105 @@ static unsigned LZ4IO_readLE32 (const void* s) unsigned value32 = srcPtr[0]; value32 += (srcPtr[1]<<8); value32 += (srcPtr[2]<<16); - value32 += (srcPtr[3]<<24); + value32 += ((unsigned)srcPtr[3])<<24; return value32; } -static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) +static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) +{ + const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ + const size_t* ptrT = bufferT; + size_t bufferSizeT = bufferSize / sizeT; + const size_t* const bufferTEnd = bufferT + bufferSizeT; + static const size_t segmentSizeT = (32 KB) / sizeT; + + if (!g_sparseFileSupport) /* normal write */ + { + size_t sizeCheck = fwrite(buffer, 1, bufferSize, file); + if (sizeCheck != bufferSize) EXM_THROW(68, "Write error : cannot write decoded block"); + return 0; + } + + /* avoid int overflow */ + if (storedSkips > 1 GB) + { + int seekResult = fseek(file, 1 GB, SEEK_CUR); + if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file support)"); + storedSkips -= 1 GB; + } + + while (ptrT < bufferTEnd) + { + size_t seg0SizeT = segmentSizeT; + size_t nb0T; + int seekResult; + + /* count leading zeros */ + if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; + bufferSizeT -= seg0SizeT; + for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; + storedSkips += (unsigned)(nb0T * sizeT); + + if (nb0T != seg0SizeT) /* not all 0s */ + { + size_t sizeCheck; + seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(68, "Skip error (sparse file)"); + storedSkips = 0; + seg0SizeT -= nb0T; + ptrT += nb0T; + sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file); + if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block"); + } + ptrT += seg0SizeT; + } + + if (bufferSize & maskT) /* size not multiple of sizeT : implies end of block */ + { + const char* const restStart = (const char*)bufferTEnd; + const char* restPtr = restStart; + size_t restSize = bufferSize & maskT; + const char* const restEnd = restStart + restSize; + for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; + storedSkips += (unsigned) (restPtr - restStart); + if (restPtr != restEnd) + { + size_t sizeCheck; + int seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(68, "Skip error (end of block)"); + storedSkips = 0; + sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); + if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block"); + } + } + + return storedSkips; +} + +static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) +{ + char lastZeroByte[1] = { 0 }; + + if (storedSkips>0) /* implies g_sparseFileSupport */ + { + int seekResult; + size_t sizeCheck; + storedSkips --; + seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); + sizeCheck = fwrite(lastZeroByte, 1, 1, file); + if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); + } +} + + +static unsigned g_magicRead = 0; +static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) { unsigned long long filesize = 0; char* in_buff; char* out_buff; + unsigned storedSkips = 0; /* Allocate Memory */ in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); @@ -556,7 +742,7 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */ if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) { /* Cannot read next block : maybe new stream ? */ - fseek(finput, -4, SEEK_CUR); + g_magicRead = blockSize; break; } @@ -570,10 +756,11 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) filesize += decodeSize; /* Write Block */ - sizeCheck = fwrite(out_buff, 1, decodeSize, foutput); - if (sizeCheck != (size_t)decodeSize) EXM_THROW(54, "Write error : cannot write decoded block into output\n"); + storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); } + LZ4IO_fwriteSparseEnd(foutput, storedSkips); + /* Free */ free(in_buff); free(out_buff); @@ -582,172 +769,129 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) } -static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) + +typedef struct { + void* srcBuffer; + size_t srcBufferSize; + void* dstBuffer; + size_t dstBufferSize; + LZ4F_decompressionContext_t dCtx; +} dRess_t; + +static const size_t LZ4IO_dBufferSize = 64 KB; + +static dRess_t LZ4IO_createDResources(void) { - unsigned long long filesize = 0; - void* inBuff; - void* outBuff; -# define HEADERMAX 20 - char headerBuff[HEADERMAX]; - size_t sizeCheck; - const size_t inBuffSize = 256 KB; - const size_t outBuffSize = 256 KB; - LZ4F_decompressionContext_t ctx; + dRess_t ress; LZ4F_errorCode_t errorCode; - unsigned storedSkips = 0; /* init */ - errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create context : %s", LZ4F_getErrorName(errorCode)); - LZ4IO_writeLE32(headerBuff, LZ4IO_MAGICNUMBER); /* regenerated here, as it was already read from finput */ + errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION); + if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); /* Allocate Memory */ - inBuff = malloc(256 KB); - outBuff = malloc(256 KB); - if (!inBuff || !outBuff) EXM_THROW(61, "Allocation error : not enough memory"); + ress.srcBufferSize = LZ4IO_dBufferSize; + ress.srcBuffer = malloc(ress.srcBufferSize); + ress.dstBufferSize = LZ4IO_dBufferSize; + ress.dstBuffer = malloc(ress.dstBufferSize); + if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); + + return ress; +} + +static void LZ4IO_freeDResources(dRess_t ress) +{ + LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx); + if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); + free(ress.srcBuffer); + free(ress.dstBuffer); +} - /* Init feed with magic number (already consumed from FILE) */ + +static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile) +{ + unsigned long long filesize = 0; + LZ4F_errorCode_t nextToLoad; + unsigned storedSkips = 0; + + /* Init feed with magic number (already consumed from FILE* sFile) */ { - size_t inSize = 4; - size_t outSize=0; - LZ4IO_writeLE32(inBuff, LZ4IO_MAGICNUMBER); - errorCode = LZ4F_decompress(ctx, outBuff, &outSize, inBuff, &inSize, NULL); - if (LZ4F_isError(errorCode)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(errorCode)); + size_t inSize = MAGICNUMBER_SIZE; + size_t outSize= 0; + LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER); + nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad)); } - /* Main Loop */ - for (;;) + for (;nextToLoad;) { size_t readSize; size_t pos = 0; + size_t decodedBytes = ress.dstBufferSize; /* Read input */ - readSize = fread(inBuff, 1, inBuffSize, finput); - if (!readSize) break; /* empty file or stream */ + if (nextToLoad > ress.srcBufferSize) nextToLoad = ress.srcBufferSize; + readSize = fread(ress.srcBuffer, 1, nextToLoad, srcFile); + if (!readSize) + break; /* empty file or stream */ - while (pos < readSize) + while ((pos < readSize) || (decodedBytes == ress.dstBufferSize)) /* still to read, or still to flush */ { /* Decode Input (at least partially) */ size_t remaining = readSize - pos; - size_t decodedBytes = outBuffSize; - errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, (char*)inBuff+pos, &remaining, NULL); - if (LZ4F_isError(errorCode)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(errorCode)); + decodedBytes = ress.dstBufferSize; + nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); pos += remaining; + if (!nextToLoad) break; if (decodedBytes) { /* Write Block */ filesize += decodedBytes; - if (g_sparseFileSupport) - { - size_t* const oBuffStartT = (size_t*)outBuff; /* since outBuff is malloc'ed, it's aligned on size_t */ - size_t* oBuffPosT = oBuffStartT; - size_t oBuffSizeT = decodedBytes / sizeT; - size_t* const oBuffEndT = oBuffStartT + oBuffSizeT; - static const size_t bs0T = (32 KB) / sizeT; - while (oBuffPosT < oBuffEndT) - { - size_t seg0SizeT = bs0T; - size_t nb0T; - int seekResult; - if (seg0SizeT > oBuffSizeT) seg0SizeT = oBuffSizeT; - oBuffSizeT -= seg0SizeT; - for (nb0T=0; (nb0T < seg0SizeT) && (oBuffPosT[nb0T] == 0); nb0T++) ; - storedSkips += (unsigned)(nb0T * sizeT); - if (storedSkips > 1 GB) /* avoid int overflow */ - { - seekResult = fseek(foutput, 1 GB, SEEK_CUR); - if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)"); - storedSkips -= 1 GB; - } - if (nb0T != seg0SizeT) /* not all 0s */ - { - seekResult = fseek(foutput, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(68, "Skip error (sparse file)"); - storedSkips = 0; - seg0SizeT -= nb0T; - oBuffPosT += nb0T; - sizeCheck = fwrite(oBuffPosT, sizeT, seg0SizeT, foutput); - if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block"); - } - oBuffPosT += seg0SizeT; - } - if (decodedBytes & maskT) /* size not multiple of sizeT (necessarily end of block) */ - { - const char* const restStart = (char*)oBuffEndT; - const char* restPtr = restStart; - size_t restSize = decodedBytes & maskT; - const char* const restEnd = restStart + restSize; - for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; - storedSkips += (unsigned) (restPtr - restStart); - if (restPtr != restEnd) - { - int seekResult = fseek(foutput, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(68, "Skip error (end of block)"); - storedSkips = 0; - sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, foutput); - if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block"); - } - } - } - else - { - sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); - if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block"); - } + DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); + storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips); } } - } - if ((g_sparseFileSupport) && (storedSkips>0)) - { - int seekResult; - storedSkips --; - seekResult = fseek(foutput, storedSkips, SEEK_CUR); - if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); - memset(outBuff, 0, 1); - sizeCheck = fwrite(outBuff, 1, 1, foutput); - if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); - } + LZ4IO_fwriteSparseEnd(dstFile, storedSkips); - /* Free */ - free(inBuff); - free(outBuff); - errorCode = LZ4F_freeDecompressionContext(ctx); - if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); + if (nextToLoad!=0) + EXM_THROW(67, "Unfinished stream"); return filesize; } -static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char U32store[MAGICNUMBER_SIZE]) +static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) { void* buffer = malloc(64 KB); size_t read = 1, sizeCheck; unsigned long long total = MAGICNUMBER_SIZE; + unsigned storedSkips = 0; - sizeCheck = fwrite(U32store, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through error at start"); + sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput); + if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error"); while (read) { read = fread(buffer, 1, 64 KB, finput); total += read; - sizeCheck = fwrite(buffer, 1, read, foutput); - if (sizeCheck != read) EXM_THROW(50, "Pass-through error"); + storedSkips = LZ4IO_fwriteSparse(foutput, buffer, read, storedSkips); } + LZ4IO_fwriteSparseEnd(foutput, storedSkips); free(buffer); return total; } #define ENDOFSTREAM ((unsigned long long)-1) -static unsigned long long selectDecoder( FILE* finput, FILE* foutput) +static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput) { - unsigned char U32store[MAGICNUMBER_SIZE]; + unsigned char MNstore[MAGICNUMBER_SIZE]; unsigned magicNumber, size; int errorNb; size_t nbReadBytes; @@ -757,33 +901,41 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput) nbCalls++; /* Check Archive Header */ - nbReadBytes = fread(U32store, 1, MAGICNUMBER_SIZE, finput); - if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */ - if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); - magicNumber = LZ4IO_readLE32(U32store); /* Little Endian format */ + if (g_magicRead) + { + magicNumber = g_magicRead; + g_magicRead = 0; + } + else + { + nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); + if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */ + if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */ + } if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ switch(magicNumber) { case LZ4IO_MAGICNUMBER: - return DEFAULT_DECOMPRESSOR(finput, foutput); + return LZ4IO_decompressLZ4F(ress, finput, foutput); case LEGACY_MAGICNUMBER: DISPLAYLEVEL(4, "Detected : Legacy format \n"); - return decodeLegacyStream(finput, foutput); + return LZ4IO_decodeLegacyStream(finput, foutput); case LZ4IO_SKIPPABLE0: DISPLAYLEVEL(4, "Skipping detected skippable area \n"); - nbReadBytes = fread(U32store, 1, 4, finput); + nbReadBytes = fread(MNstore, 1, 4, finput); if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); - size = LZ4IO_readLE32(U32store); /* Little Endian format */ + size = LZ4IO_readLE32(MNstore); /* Little Endian format */ errorNb = fseek(finput, size, SEEK_CUR); if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); - return selectDecoder(finput, foutput); + return selectDecoder(ress, finput, foutput); EXTENDED_FORMAT; default: if (nbCalls == 1) /* just started */ { if (g_overwrite) - return LZ4IO_passThrough(finput, foutput, U32store); + return LZ4IO_passThrough(finput, foutput, MNstore); EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ } DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); @@ -792,43 +944,95 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput) } -int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) +static int LZ4IO_decompressFile_extRess(dRess_t ress, const char* input_filename, const char* output_filename) { unsigned long long filesize = 0, decodedSize=0; FILE* finput; FILE* foutput; - clock_t start, end; /* Init */ - start = clock(); - get_fileHandle(input_filename, output_filename, &finput, &foutput); + if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput)) + return 1; /* sparse file */ - if (g_sparseFileSupport && foutput) { SET_SPARSE_FILE_MODE(foutput); } + if (g_sparseFileSupport) { SET_SPARSE_FILE_MODE(foutput); } /* Loop over multiple streams */ do { - decodedSize = selectDecoder(finput, foutput); + decodedSize = selectDecoder(ress, finput, foutput); if (decodedSize != ENDOFSTREAM) filesize += decodedSize; } while (decodedSize != ENDOFSTREAM); /* Final Status */ - end = clock(); DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } /* Close */ fclose(finput); fclose(foutput); - /* Error status = OK */ return 0; } + +int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) +{ + dRess_t ress; + clock_t start, end; + int missingFiles = 0; + + start = clock(); + + ress = LZ4IO_createDResources(); + missingFiles += LZ4IO_decompressFile_extRess(ress, input_filename, output_filename); + LZ4IO_freeDResources(ress); + + end = clock(); + if (end==start) end=start+1; + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds); + } + + return missingFiles; +} + + +int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) +{ + int i; + int skippedFiles = 0; + int missingFiles = 0; + char* outFileName = (char*)malloc(FNSPACE); + size_t ofnSize = FNSPACE; + const size_t suffixSize = strlen(suffix); + char* ifnSuffix = (char*)malloc(suffixSize + 1); + dRess_t ress; + + ress = LZ4IO_createDResources(); + + for (i=0; i<ifntSize; i++) + { + size_t ifnSize = strlen(inFileNamesTable[i]); + strcpy(ifnSuffix, inFileNamesTable[i] + ifnSize - suffixSize); + if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); } + if (ifnSize <= suffixSize || strcmp(ifnSuffix, suffix) != 0) + { + DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); + skippedFiles++; + continue; + } + memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize); + outFileName[ifnSize-suffixSize] = '\0'; + + missingFiles += LZ4IO_decompressFile_extRess(ress, inFileNamesTable[i], outFileName); + } + + LZ4IO_freeDResources(ress); + free(outFileName); + free(ifnSuffix); + return missingFiles + skippedFiles; +} diff --git a/programs/lz4io.h b/programs/lz4io.h index 75a36e1..11c8fdb 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -51,8 +51,9 @@ static char const nulmark[] = "/dev/null"; int LZ4IO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel); int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename); -int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); +int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); +int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix); /* ************************************************** */ /* ****************** Parameters ******************** */ |