summaryrefslogtreecommitdiffstats
path: root/programs
diff options
context:
space:
mode:
authorYann Collet <yann.collet.73@gmail.com>2015-05-07 11:23:39 (GMT)
committerYann Collet <yann.collet.73@gmail.com>2015-05-07 11:23:39 (GMT)
commit7d182b816ace89d6d6d16b7aee376a9962a05caa (patch)
tree5b2236b59cd0e42e3365b0765041a25c85519b39 /programs
parent160661c7a4cbf805f4af74d2e3932a17a66e6ce7 (diff)
parentfdd0029c3778768d0e8a80ce238d981f946c8b78 (diff)
downloadlz4-7d182b816ace89d6d6d16b7aee376a9962a05caa.zip
lz4-7d182b816ace89d6d6d16b7aee376a9962a05caa.tar.gz
lz4-7d182b816ace89d6d6d16b7aee376a9962a05caa.tar.bz2
Merge pull request #102 from Cyan4973/devrc129v0r129
Dev
Diffstat (limited to 'programs')
-rw-r--r--programs/Makefile118
-rw-r--r--programs/bench.c31
-rw-r--r--programs/frametest.c152
-rw-r--r--programs/fullbench.c454
-rw-r--r--programs/fuzzer.c269
-rw-r--r--programs/lz4.114
-rw-r--r--programs/lz4cli.c72
-rw-r--r--programs/lz4io.c672
-rw-r--r--programs/lz4io.h3
9 files changed, 1119 insertions, 666 deletions
diff --git a/programs/Makefile b/programs/Makefile
index a324148..39335db 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -27,24 +27,24 @@
# lz4c32: Same as lz4c, but forced to compile in 32-bits mode
# fuzzer : Test tool, to check lz4 integrity on target platform
# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
+# frametest : Test tool, to check lz4frame integrity on target platform
+# frametest32: Same as frametest, but forced to compile in 32-bits mode
# fullbench : Precisely measure speed for each LZ4 function variant
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
+# datagen : generates synthetic data samples for tests & benchmarks
# ##########################################################################
-RELEASE?= r128
+RELEASE?= r129
DESTDIR?=
PREFIX ?= /usr/local
CFLAGS ?= -O3
-CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\"
-FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\"
+FLAGS := -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
-BINDIR=$(PREFIX)/bin
-MANDIR=$(PREFIX)/share/man/man1
-LZ4DIR=../lib
-
-TEST_FILES = COPYING
-TEST_TARGETS=test-native
+BINDIR := $(PREFIX)/bin
+MANDIR := $(PREFIX)/share/man/man1
+LZ4DIR := ../lib
# Define *.exe as extension for Windows systems
@@ -58,7 +58,10 @@ endif
# Select test target for Travis CI's Build Matrix
-TRAVIS_TARGET=$(LZ4_TRAVIS_CI_ENV)
+TRAVIS_TARGET:= $(LZ4_TRAVIS_CI_ENV)
+TEST_FILES := COPYING
+TEST_TARGETS := test-native
+FUZZER_TIME := -T9mn
default: lz4
@@ -100,7 +103,7 @@ datagen : datagen.c datagencli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
clean:
- @rm -f core *.o *.test \
+ @rm -f core *.o *.test tmp* \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) \
@@ -148,15 +151,18 @@ test-travis: $(TRAVIS_TARGET)
test-lz4-sparse: lz4 datagen
@echo ---- test sparse file support ----
- ./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dv --sparse > tmpB4
- ./datagen -g50M -P100 | ./lz4 -B5D | ./lz4 -dv --sparse > tmpB5
- ./datagen -g50M -P100 | ./lz4 -B6D | ./lz4 -dv --sparse > tmpB6
- ./datagen -g50M -P100 | ./lz4 -B7D | ./lz4 -dv --sparse > tmpB7
+ ./datagen -g5M -P100 > tmpSrc
+ ./lz4 -B4D tmpSrc | ./lz4 -dv --sparse > tmpB4
+ diff -s tmpSrc tmpB4
+ ./lz4 -B5D tmpSrc | ./lz4 -dv --sparse > tmpB5
+ diff -s tmpSrc tmpB5
+ ./lz4 -B6D tmpSrc | ./lz4 -dv --sparse > tmpB6
+ diff -s tmpSrc tmpB6
+ ./lz4 -B7D tmpSrc | ./lz4 -dv --sparse > tmpB7
+ diff -s tmpSrc tmpB7
+ ./lz4 tmpSrc | ./lz4 -dv --no-sparse > tmpNoSparse
+ diff -s tmpSrc tmpNoSparse
ls -ls tmp*
- ./datagen -g50M -P100 | diff -s - tmpB4
- ./datagen -g50M -P100 | diff -s - tmpB5
- ./datagen -g50M -P100 | diff -s - tmpB6
- ./datagen -g50M -P100 | diff -s - tmpB7
./datagen -s1 -g1200007 -P100 | ./lz4 | ./lz4 -dv --sparse > tmpOdd # Odd size file (to not finish on an exact nb of blocks)
./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd
ls -ls tmpOdd
@@ -168,6 +174,11 @@ test-lz4-contentSize: lz4 datagen
./lz4 -v tmp | ./lz4 -t
./lz4 -v --content-size tmp | ./lz4 -d > tmp2
diff -s tmp tmp2
+ # test large size [2-4] GB
+ @./datagen -g3G -P100 | ./lz4 | ./lz4 --decompress --force --sparse - tmp
+ @ls -ls tmp
+ ./lz4 --quiet --content-size tmp | ./lz4 --verbose --decompress --force --sparse - tmp2
+ @ls -ls tmp2
@rm tmp*
test-lz4-frame-concatenation: lz4 datagen
@@ -183,24 +194,30 @@ test-lz4-frame-concatenation: lz4 datagen
@rm *.test
@echo frame concatenation test completed
-test-lz4: lz4 datagen test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
+test-lz4-multiple: lz4 datagen
+ @echo ---- test multiple files ----
+ @./datagen -s1 > tmp1 2> $(VOID)
+ @./datagen -s2 -g100K > tmp2 2> $(VOID)
+ @./datagen -s3 -g1M > tmp3 2> $(VOID)
+ ./lz4 -f -m tmp*
+ ls -ls tmp*
+ rm tmp1 tmp2 tmp3
+ ./lz4 -df -m *.lz4
+ ls -ls tmp*
+ ./lz4 -f -m tmp1 notHere tmp2; echo $$?
+ @rm tmp*
+
+test-lz4: lz4 datagen test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
@echo ---- test lz4 basic compression/decompression ----
./datagen -g0 | ./lz4 -v | ./lz4 -t
./datagen -g16KB | ./lz4 -9 | ./lz4 -t
./datagen | ./lz4 | ./lz4 -t
./datagen -g6M -P99 | ./lz4 -9BD | ./lz4 -t
- ./datagen -g17M | ./lz4 -9v | ./lz4 -tq
+ ./datagen -g17M | ./lz4 -9v | ./lz4 -qt
./datagen -g33M | ./lz4 --no-frame-crc | ./lz4 -t
./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t
- ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -t
- ./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -t
- @echo ---- test multiple input files ----
- @./datagen -s1 > file1
- @./datagen -s2 > file2
- @./datagen -s3 > file3
- ./lz4 -f -m file1 file2 file3
- ls -l file*
- @rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4
+ ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -qt
+ ./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -qt
@echo ---- test pass-through ----
./datagen | ./lz4 -tf
@@ -212,41 +229,46 @@ test-lz4c32: lz4 lz4c32 datagen
./datagen -g16KB | ./lz4c32 -9 | ./lz4 -t
./datagen | ./lz4c32 | ./lz4c32 -t
./datagen | ./lz4c32 | ./lz4 -t
- ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -t
- ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -t
- ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -t
- ./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -t
+ ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -qt
+ ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -qt
+ ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -qt
+ ./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -qt
test-fullbench: fullbench
- ./fullbench --no-prompt $(TEST_FILES)
+ ./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES)
test-fullbench32: fullbench32
- ./fullbench32 --no-prompt $(TEST_FILES)
+ ./fullbench32 --no-prompt $(NB_LOOPS) $(TEST_FILES)
test-fuzzer: fuzzer
- ./fuzzer
+ ./fuzzer $(FUZZER_TIME)
test-fuzzer32: fuzzer32
- ./fuzzer32
+ ./fuzzer32 $(FUZZER_TIME)
test-frametest: frametest
- ./frametest
+ ./frametest $(FUZZER_TIME)
test-frametest32: frametest32
- ./frametest32
+ ./frametest32 $(FUZZER_TIME)
-test-mem: lz4 datagen fuzzer frametest
- valgrind --leak-check=yes ./datagen -g50M > $(VOID)
+test-mem: lz4 datagen fuzzer frametest fullbench
+ valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
./datagen -g16KB > tmp
- valgrind --leak-check=yes ./lz4 -9 -BD -f tmp $(VOID)
+ valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -BD -f tmp $(VOID)
+ ./datagen -g16KB -s2 > tmp2
+ ./datagen -g16KB -s3 > tmp3
+ valgrind --leak-check=yes --error-exitcode=1 ./lz4 --force --multiple tmp tmp2 tmp3
./datagen -g16MB > tmp
- valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp tmp2
- valgrind --leak-check=yes ./lz4 -t tmp2
+ valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -B5D -f tmp tmp2
+ valgrind --leak-check=yes --error-exitcode=1 ./lz4 -t tmp2
+ valgrind --leak-check=yes --error-exitcode=1 ./lz4 -bi1 tmp
+ valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2
./datagen -g256MB > tmp
- valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp $(VOID)
+ valgrind --leak-check=yes --error-exitcode=1 ./lz4 -B4D -f -vq tmp $(VOID)
rm tmp*
- valgrind --leak-check=yes ./fuzzer -i64 -t1
- valgrind --leak-check=yes ./frametest -i256
+ valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
+ valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256
test-mem32: lz4c32 datagen
# unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system...
diff --git a/programs/bench.c b/programs/bench.c
index e1b5357..9f949c4 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -58,9 +58,9 @@
#include "lz4.h"
#define COMPRESSOR0 LZ4_compress_local
-static int LZ4_compress_local(const char* src, char* dst, int size, int clevel) { (void)clevel; return LZ4_compress(src, dst, size); }
+static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSize, int clevel) { (void)clevel; return LZ4_compress_default(src, dst, srcSize, dstSize); }
#include "lz4hc.h"
-#define COMPRESSOR1 LZ4_compressHC2
+#define COMPRESSOR1 LZ4_compress_HC
#define DEFAULTCOMPRESSOR COMPRESSOR0
#include "xxhash.h"
@@ -121,8 +121,8 @@ struct chunkParameters
struct compressionParameters
{
- int (*compressionFunction)(const char*, char*, int, int);
- int (*decompressionFunction)(const char*, char*, int);
+ int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
+ int (*decompressionFunction)(const char* src, char* dst, int dstSize);
};
@@ -281,17 +281,13 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
/* Check file existence */
inFileName = fileNamesTable[fileIdx++];
inFile = fopen( inFileName, "rb" );
- if (inFile==NULL)
- {
- DISPLAY( "Pb opening %s\n", inFileName);
- return 11;
- }
+ if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
/* Memory allocation & restrictions */
inFileSize = BMK_GetFileSize(inFileName);
- if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; }
+ if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; }
benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2;
- if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; }
+ if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; }
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
{
@@ -306,7 +302,6 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
compressedBuffSize = nbChunks * maxCompressedChunkSize;
compressedBuffer = (char*)malloc((size_t)compressedBuffSize);
-
if (!orig_buff || !compressedBuffer)
{
DISPLAY("\nError: not enough memory!\n");
@@ -376,11 +371,12 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
- chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, cLevel);
+ chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize, cLevel);
nbLoops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
+ nbLoops += !nbLoops; /* avoid division by zero */
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime/nbLoops;
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
ratio = (double)cSize/(double)benchedSize*100.;
@@ -402,8 +398,9 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
}
milliTime = BMK_GetMilliSpan(milliTime);
+ nbLoops += !nbLoops; /* avoid division by zero */
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime/nbLoops;
- DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
+ DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
/* CRC Checking */
crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0);
@@ -413,9 +410,9 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
if (crcOrig==crcCheck)
{
if (ratio<100.)
- DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
+ DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
else
- DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
+ DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
}
totals += benchedSize;
totalz += cSize;
@@ -431,7 +428,7 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
if (nbFiles > 1)
DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.);
- if (BMK_pause) { DISPLAY("\npress enter...\n"); getchar(); }
+ if (BMK_pause) { DISPLAY("\npress enter...\n"); (void)getchar(); }
return 0;
}
diff --git a/programs/frametest.c b/programs/frametest.c
index 4beee23..46ec030 100644
--- a/programs/frametest.c
+++ b/programs/frametest.c
@@ -238,10 +238,11 @@ int basicTests(U32 seed, double compressibility)
U32 randState = seed;
size_t cSize, testSize;
LZ4F_preferences_t prefs;
- LZ4F_decompressionContext_t dCtx;
+ LZ4F_decompressionContext_t dCtx = NULL;
+ LZ4F_compressionContext_t cctx = NULL;
U64 crcOrig;
- // Create compressible test buffer
+ /* Create compressible test buffer */
memset(&prefs, 0, sizeof(prefs));
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL));
@@ -249,7 +250,7 @@ int basicTests(U32 seed, double compressibility)
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
- // Trivial tests : one-step frame
+ /* Trivial tests : one-step frame */
testSize = COMPRESSIBLE_NOISE_LENGTH;
DISPLAYLEVEL(3, "Using NULL preferences : \n");
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
@@ -276,12 +277,58 @@ int basicTests(U32 seed, double compressibility)
if (crcDest != crcOrig) goto _output_error;
DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
+ DISPLAYLEVEL(4, "Reusing decompression context \n");
+ {
+ size_t iSize = compressedBufferSize - 4;
+ const BYTE* cBuff = (const BYTE*) compressedBuffer;
+ DISPLAYLEVEL(3, "Missing last 4 bytes : ");
+ errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL);
+ if (LZ4F_isError(errorCode)) goto _output_error;
+ if (!errorCode) goto _output_error;
+ DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned)errorCode);
+ cBuff += iSize;
+ iSize = errorCode;
+ errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL);
+ if (errorCode != 0) goto _output_error;
+ crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
+ if (crcDest != crcOrig) goto _output_error;
+ }
+
+ {
+ size_t oSize = 0;
+ size_t iSize = 0;
+ LZ4F_frameInfo_t fi;
+
+ DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
+ errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
+ if (LZ4F_isError(errorCode)) goto _output_error;
+ DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
+
+ DISPLAYLEVEL(3, "get FrameInfo on null input : ");
+ errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
+ if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
+ DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
+
+ DISPLAYLEVEL(3, "get FrameInfo on not enough input : ");
+ iSize = 6;
+ errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
+ if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
+ DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
+ ip += iSize;
+
+ DISPLAYLEVEL(3, "get FrameInfo on enough input : ");
+ iSize = 15 - iSize;
+ errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
+ if (LZ4F_isError(errorCode)) goto _output_error;
+ DISPLAYLEVEL(3, " correctly decoded \n");
+ ip += iSize;
+ }
+
DISPLAYLEVEL(3, "Byte after byte : \n");
while (ip < iend)
{
size_t oSize = oend-op;
size_t iSize = 1;
- //DISPLAY("%7i \n", (int)(ip-(BYTE*)compressedBuffer));
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += oSize;
@@ -289,28 +336,28 @@ int basicTests(U32 seed, double compressibility)
}
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (crcDest != crcOrig) goto _output_error;
- DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
+ DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-(BYTE*)decodedBuffer), COMPRESSIBLE_NOISE_LENGTH);
errorCode = LZ4F_freeDecompressionContext(dCtx);
if (LZ4F_isError(errorCode)) goto _output_error;
}
DISPLAYLEVEL(3, "Using 64 KB block : \n");
- prefs.frameInfo.blockSizeID = max64KB;
- prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
+ prefs.frameInfo.blockSizeID = LZ4F_max64KB;
+ prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : \n");
- prefs.frameInfo.contentChecksumFlag = noContentChecksum;
+ prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 256 KB block : \n");
- prefs.frameInfo.blockSizeID = max256KB;
- prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
+ prefs.frameInfo.blockSizeID = LZ4F_max256KB;
+ prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
@@ -350,33 +397,33 @@ int basicTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "without checksum : \n");
- prefs.frameInfo.contentChecksumFlag = noContentChecksum;
+ prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 1 MB block : \n");
- prefs.frameInfo.blockSizeID = max1MB;
- prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
+ prefs.frameInfo.blockSizeID = LZ4F_max1MB;
+ prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : \n");
- prefs.frameInfo.contentChecksumFlag = noContentChecksum;
+ prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 4 MB block : \n");
- prefs.frameInfo.blockSizeID = max4MB;
- prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
+ prefs.frameInfo.blockSizeID = LZ4F_max4MB;
+ prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : \n");
- prefs.frameInfo.contentChecksumFlag = noContentChecksum;
+ prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
@@ -385,7 +432,6 @@ int basicTests(U32 seed, double compressibility)
size_t errorCode;
BYTE* const ostart = (BYTE*)compressedBuffer;
BYTE* op = ostart;
- LZ4F_compressionContext_t cctx;
errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error;
@@ -430,6 +476,7 @@ int basicTests(U32 seed, double compressibility)
errorCode = LZ4F_freeCompressionContext(cctx);
if (LZ4F_isError(errorCode)) goto _output_error;
+ cctx = NULL;
}
DISPLAYLEVEL(3, "Skippable frame test : \n");
@@ -500,10 +547,6 @@ int basicTests(U32 seed, double compressibility)
ip += iSize;
}
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
-
- /* release memory */
- errorCode = LZ4F_freeDecompressionContext(dCtx);
- if (LZ4F_isError(errorCode)) goto _output_error;
}
DISPLAY("Basic tests completed \n");
@@ -511,6 +554,8 @@ _end:
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
+ LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
+ LZ4F_freeCompressionContext(cctx); cctx = NULL;
return testResult;
_output_error:
@@ -523,8 +568,8 @@ _output_error:
static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
{
int p=0;
- BYTE* b1=(BYTE*)buff1;
- BYTE* b2=(BYTE*)buff2;
+ const BYTE* b1=(const BYTE*)buff1;
+ const BYTE* b2=(const BYTE*)buff2;
if (nonContiguous)
{
DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size);
@@ -537,7 +582,7 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un
static const U32 srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */
-int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility)
+int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration)
{
unsigned testResult = 0;
unsigned testNb = 0;
@@ -548,10 +593,15 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
LZ4F_decompressionContext_t dCtx = NULL;
LZ4F_compressionContext_t cCtx = NULL;
size_t result;
+ const U32 startTime = FUZ_GetMilliStart();
XXH64_state_t xxh64;
# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
+
+ /* Init */
+ duration *= 1000;
+
/* Create buffers */
result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
@@ -566,10 +616,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);
/* jump to requested testNb */
- for (testNb =0; testNb < startTest; testNb++) (void)FUZ_rand(&coreRand); // sync randomizer
+ for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer
/* main fuzzer test loop */
- for ( ; testNb < nbTests; testNb++)
+ for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++)
{
U32 randState = coreRand ^ prime1;
unsigned BSId = 4 + (FUZ_rand(&randState) & 3);
@@ -591,9 +641,9 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
memset(&prefs, 0, sizeof(prefs));
memset(&cOptions, 0, sizeof(cOptions));
memset(&dOptions, 0, sizeof(dOptions));
- prefs.frameInfo.blockMode = (blockMode_t)BMId;
- prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId;
- prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag;
+ prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId;
+ prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId;
+ prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag;
prefs.frameInfo.contentSize = frameContentSize;
prefs.autoFlush = autoflush;
prefs.compressionLevel = FUZ_rand(&randState) % 5;
@@ -662,6 +712,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1;
nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */
XXH64_reset(&xxh64, 1);
+ if (maxBits < 3) maxBits = 3;
while (ip < iend)
{
unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
@@ -673,7 +724,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
dOptions.stableDst = FUZ_rand(&randState) & 1;
if (nonContiguousDst==2) dOptions.stableDst = 0;
result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
- if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
+ if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid)
+ locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result));
XXH64_update(&xxh64, op, (U32)oSize);
totalOut += oSize;
@@ -704,7 +756,7 @@ _end:
if (pause)
{
DISPLAY("press enter to finish \n");
- getchar();
+ (void)getchar();
}
return testResult;
@@ -721,6 +773,7 @@ int FUZ_usage(void)
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
+ DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
DISPLAY( " -s# : Select seed (default:prompt user)\n");
DISPLAY( " -t# : Select starting test number (default:0)\n");
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
@@ -739,6 +792,7 @@ int main(int argc, char** argv)
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int result=0;
+ U32 duration=0;
/* Check command line */
programName = argv[0];
@@ -746,9 +800,9 @@ int main(int argc, char** argv)
{
char* argument = argv[argNb];
- if(!argument) continue; // Protection if argument empty
+ if(!argument) continue; /* Protection if argument empty */
- // Decode command (note : aggregated commands are allowed)
+ /* Decode command (note : aggregated commands are allowed) */
if (argument[0]=='-')
{
if (!strcmp(argument, "--no-prompt"))
@@ -781,7 +835,7 @@ int main(int argc, char** argv)
case 'i':
argument++;
- nbTests=0;
+ nbTests=0; duration=0;
while ((*argument>='0') && (*argument<='9'))
{
nbTests *= 10;
@@ -789,6 +843,32 @@ int main(int argc, char** argv)
argument++;
}
break;
+
+ case 'T':
+ argument++;
+ nbTests = 0; duration = 0;
+ for (;;)
+ {
+ switch(*argument)
+ {
+ case 'm': duration *= 60; argument++; continue;
+ case 's':
+ case 'n': argument++; continue;
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9': duration *= 10; duration += *argument++ - '0'; continue;
+ }
+ break;
+ }
+ break;
+
case 's':
argument++;
seed=0;
@@ -841,5 +921,5 @@ int main(int argc, char** argv)
if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
if (result) return 1;
- return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
+ return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration);
}
diff --git a/programs/fullbench.c b/programs/fullbench.c
index 0c6e05e..0d08a40 100644
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -30,7 +30,7 @@
#define _CRT_SECURE_NO_WARNINGS
#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
-// Unix Large Files support (>4GB)
+/* Unix Large Files support (>4GB) */
#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
# define _LARGEFILE_SOURCE
# define _FILE_OFFSET_BITS 64
@@ -47,17 +47,17 @@
/**************************************
* Includes
**************************************/
-#include <stdlib.h> // malloc
-#include <stdio.h> // fprintf, fopen, ftello64
-#include <sys/types.h> // stat64
-#include <sys/stat.h> // stat64
-#include <string.h> // strcmp
+#include <stdlib.h> /* malloc, free */
+#include <stdio.h> /* fprintf, fopen, ftello64 */
+#include <sys/types.h> /* stat64 */
+#include <sys/stat.h> /* stat64 */
+#include <string.h> /* strcmp */
-// Use ftime() if gettimeofday() is not available on your target
+/* Use ftime() if gettimeofday() is not available on your target */
#if defined(BMK_LEGACY_TIMER)
-# include <sys/timeb.h> // timeb, ftime
+# include <sys/timeb.h> /* timeb, ftime */
#else
-# include <sys/time.h> // gettimeofday
+# include <sys/time.h> /* gettimeofday */
#endif
#include "lz4.h"
@@ -75,16 +75,11 @@
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
-// GCC does not support _rotl outside of Windows
-#if !defined(_WIN32)
-# define _rotl(x,r) ((x << r) | (x >> (32 - r)))
-#endif
-
/**************************************
* Basic Types
**************************************/
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
@@ -118,7 +113,7 @@
#define GB *(1U<<30)
#define KNUTH 2654435761U
-#define MAX_MEM (1984 MB)
+#define MAX_MEM (1920 MB)
#define DEFAULT_CHUNKSIZE (4 MB)
#define ALL_COMPRESSORS 0
@@ -139,53 +134,53 @@ struct chunkParameters
/**************************************
-* MACRO
+* Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-#define PROGRESS(...) no_prompt ? 0 : DISPLAY(__VA_ARGS__)
-
+#define PROGRESS(...) g_noPrompt ? 0 : DISPLAY(__VA_ARGS__)
-//**************************************
-// Benchmark Parameters
-//**************************************
-static int chunkSize = DEFAULT_CHUNKSIZE;
-static int nbIterations = NBLOOPS;
-static int BMK_pause = 0;
-static int compressionTest = 1;
-static int decompressionTest = 1;
-static int compressionAlgo = ALL_COMPRESSORS;
-static int decompressionAlgo = ALL_DECOMPRESSORS;
-static int no_prompt = 0;
+/**************************************
+* Benchmark Parameters
+**************************************/
+static int g_chunkSize = DEFAULT_CHUNKSIZE;
+static int g_nbIterations = NBLOOPS;
+static int g_pause = 0;
+static int g_compressionTest = 1;
+static int g_compressionAlgo = ALL_COMPRESSORS;
+static int g_decompressionTest = 1;
+static int g_decompressionAlgo = ALL_DECOMPRESSORS;
+static int g_noPrompt = 0;
-void BMK_SetBlocksize(int bsize)
+static void BMK_setBlocksize(int bsize)
{
- chunkSize = bsize;
- DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10);
+ g_chunkSize = bsize;
+ DISPLAY("-Using Block Size of %i KB-\n", g_chunkSize>>10);
}
-void BMK_SetNbIterations(int nbLoops)
+static void BMK_setNbIterations(int nbLoops)
{
- nbIterations = nbLoops;
- DISPLAY("- %i iterations -\n", nbIterations);
+ g_nbIterations = nbLoops;
+ DISPLAY("- %i iterations -\n", g_nbIterations);
}
-void BMK_SetPause(void)
+static void BMK_setPause(void)
{
- BMK_pause = 1;
+ g_pause = 1;
}
-//*********************************************************
-// Private functions
-//*********************************************************
+
+/*********************************************************
+* Private functions
+*********************************************************/
#if defined(BMK_LEGACY_TIMER)
static int BMK_GetMilliStart(void)
{
- // Based on Legacy ftime()
- // Rolls over every ~ 12.1 days (0x100000/24/60/60)
- // Use GetMilliSpan to correct for rollover
+ /* Based on Legacy ftime()
+ * Rolls over every ~ 12.1 days (0x100000/24/60/60)
+ * Use GetMilliSpan to correct for rollover */
struct timeb tb;
int nCount;
ftime( &tb );
@@ -197,8 +192,8 @@ static int BMK_GetMilliStart(void)
static int BMK_GetMilliStart(void)
{
- // Based on newer gettimeofday()
- // Use GetMilliSpan to correct for rollover
+ /* Based on newer gettimeofday()
+ * Use GetMilliSpan to correct for rollover */
struct timeval tv;
int nCount;
gettimeofday(&tv, NULL);
@@ -253,7 +248,7 @@ static U64 BMK_GetFileSize(char* infilename)
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
- if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
return (U64)statbuf.st_size;
}
@@ -381,106 +376,141 @@ start:
#endif // __SSSE3__
-static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
+static LZ4_stream_t LZ4_stream;
+static void local_LZ4_resetDictT(void)
{
- return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
+ LZ4_resetStream(&LZ4_stream);
}
-static void* stateLZ4;
-static int local_LZ4_compress_withState(const char* in, char* out, int inSize)
+static void local_LZ4_createStream(void)
{
- return LZ4_compress_withState(stateLZ4, in, out, inSize);
+ LZ4_resetStream(&LZ4_stream);
}
-static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
+static int local_LZ4_saveDict(const char* in, char* out, int inSize)
{
- return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize));
+ (void)in;
+ return LZ4_saveDict(&LZ4_stream, out, inSize);
}
-static LZ4_stream_t* ctx;
-static int local_LZ4_compress_continue(const char* in, char* out, int inSize)
+static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
{
- return LZ4_compress_continue(ctx, in, out, inSize);
+ return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1);
}
-static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
+static int local_LZ4_compress_default_large(const char* in, char* out, int inSize)
{
- return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
+ return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize));
}
+static int local_LZ4_compress_default_small(const char* in, char* out, int inSize)
+{
+ return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1);
+}
-LZ4_stream_t LZ4_dict;
-static void* local_LZ4_resetDictT(const char* fake)
+static int local_LZ4_compress_fast0(const char* in, char* out, int inSize)
{
- (void)fake;
- memset(&LZ4_dict, 0, sizeof(LZ4_stream_t));
- return NULL;
+ return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 0);
}
-int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize);
-static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
+static int local_LZ4_compress_fast1(const char* in, char* out, int inSize)
{
- return LZ4_compress_forceExtDict(&LZ4_dict, in, out, inSize);
+ return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 1);
}
+static int local_LZ4_compress_fast2(const char* in, char* out, int inSize)
+{
+ return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 2);
+}
-static void* stateLZ4HC;
-static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
+static int local_LZ4_compress_fast17(const char* in, char* out, int inSize)
{
- return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize);
+ return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 17);
}
-static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
+static int local_LZ4_compress_fast_extState0(const char* in, char* out, int inSize)
{
- return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
+ return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0);
}
-static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
+static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSize)
{
- return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
+ return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0);
}
-static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
+static int local_LZ4_compress_withState(const char* in, char* out, int inSize)
{
- return LZ4_compressHC_continue((LZ4_streamHC_t*)ctx, in, out, inSize);
+ return LZ4_compress_withState(&LZ4_stream, in, out, inSize);
}
-static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
+static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
{
- return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, in, out, inSize, LZ4_compressBound(inSize));
+ return LZ4_compress_limitedOutput_withState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1);
}
-static int local_LZ4F_compressFrame(const char* in, char* out, int inSize)
+static int local_LZ4_compress_continue(const char* in, char* out, int inSize)
{
- return (int)LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL);
+ return LZ4_compress_continue(&LZ4_stream, in, out, inSize);
}
-static int local_LZ4_saveDict(const char* in, char* out, int inSize)
+static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
{
- (void)in;
- return LZ4_saveDict(&LZ4_dict, out, inSize);
+ return LZ4_compress_limitedOutput_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1);
+}
+
+/* declare hidden function */
+int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
+
+static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
+{
+ return LZ4_compress_forceExtDict(&LZ4_stream, in, out, inSize);
+}
+
+
+/* HC compression functions */
+LZ4_streamHC_t LZ4_streamHC;
+static void local_LZ4_resetStreamHC(void)
+{
+ LZ4_resetStreamHC(&LZ4_streamHC, 0);
}
-LZ4_streamHC_t LZ4_dictHC;
static int local_LZ4_saveDictHC(const char* in, char* out, int inSize)
{
(void)in;
- return LZ4_saveDictHC(&LZ4_dictHC, out, inSize);
+ return LZ4_saveDictHC(&LZ4_streamHC, out, inSize);
}
+static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
+{
+ return LZ4_compressHC_withStateHC(&LZ4_streamHC, in, out, inSize);
+}
-static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
+static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
{
- (void)inSize;
- //lz4_decode_sse((BYTE*)out, (BYTE*)in, inSize);
- LZ4_decompress_fast(in, out, outSize);
- return outSize;
+ return LZ4_compressHC_limitedOutput_withStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1);
+}
+
+static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
+{
+ return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1);
+}
+
+static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
+{
+ return LZ4_compressHC_continue(&LZ4_streamHC, in, out, inSize);
}
-static int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
+static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
+{
+ return LZ4_compressHC_limitedOutput_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1);
+}
+
+
+/* decompression functions */
+static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
{
(void)inSize;
- LZ4_decompress_fast_withPrefix64k(in, out, outSize);
+ LZ4_decompress_fast(in, out, outSize);
return outSize;
}
@@ -512,6 +542,13 @@ static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSi
return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
}
+
+/* frame functions */
+static int local_LZ4F_compressFrame(const char* in, char* out, int inSize)
+{
+ return (int)LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL);
+}
+
static LZ4F_decompressionContext_t g_dCtx;
static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outSize)
@@ -526,83 +563,65 @@ static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outS
}
+#define NB_COMPRESSION_ALGORITHMS 100
+#define NB_DECOMPRESSION_ALGORITHMS 100
int fullSpeedBench(char** fileNamesTable, int nbFiles)
{
int fileIdx=0;
- char* orig_buff;
-# define NB_COMPRESSION_ALGORITHMS 16
- double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0};
- double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0};
-# define NB_DECOMPRESSION_ALGORITHMS 9
- double totalDTime[NB_DECOMPRESSION_ALGORITHMS+1] = {0};
size_t errorCode;
+ /* Init */
errorCode = LZ4F_createDecompressionContext(&g_dCtx, LZ4F_VERSION);
- if (LZ4F_isError(errorCode))
- {
- DISPLAY("dctx allocation issue \n");
- return 10;
- }
+ if (LZ4F_isError(errorCode)) { DISPLAY("dctx allocation issue \n"); return 10; }
- // Loop for each file
+ /* Loop for each fileName */
while (fileIdx<nbFiles)
{
FILE* inFile;
+ char* orig_buff = NULL;
+ struct chunkParameters* chunkP = NULL;
+ char* compressed_buff=NULL;
char* inFileName;
U64 inFileSize;
size_t benchedSize;
int nbChunks;
int maxCompressedChunkSize;
- struct chunkParameters* chunkP;
size_t readSize;
- char* compressed_buff; int compressedBuffSize;
+ int compressedBuffSize;
U32 crcOriginal;
-
- // Init
- stateLZ4 = LZ4_createStream();
- stateLZ4HC = LZ4_createStreamHC();
-
- // Check file existence
+ /* Check file existence */
inFileName = fileNamesTable[fileIdx++];
inFile = fopen( inFileName, "rb" );
- if (inFile==NULL)
- {
- DISPLAY( "Pb opening %s\n", inFileName);
- return 11;
- }
+ if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
- // Memory allocation & restrictions
+ /* Memory size adjustments */
inFileSize = BMK_GetFileSize(inFileName);
- if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; }
- benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2;
- if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; }
+ if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; }
+ benchedSize = (size_t) BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */
+ if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; }
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
- {
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
- }
- // Alloc
- chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)chunkSize)+1) * sizeof(struct chunkParameters));
- orig_buff = (char*) malloc((size_t)benchedSize);
- nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize);
- maxCompressedChunkSize = LZ4_compressBound(chunkSize);
+ /* Allocation */
+ chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)g_chunkSize)+1) * sizeof(struct chunkParameters));
+ orig_buff = (char*) malloc(benchedSize);
+ nbChunks = (int) ((benchedSize + (g_chunkSize-1)) / g_chunkSize);
+ maxCompressedChunkSize = LZ4_compressBound(g_chunkSize);
compressedBuffSize = nbChunks * maxCompressedChunkSize;
compressed_buff = (char*)malloc((size_t)compressedBuffSize);
-
-
- if(!orig_buff || !compressed_buff)
+ if(!chunkP || !orig_buff || !compressed_buff)
{
- DISPLAY("\nError: not enough memory!\n");
- free(orig_buff);
- free(compressed_buff);
- free(chunkP);
- fclose(inFile);
- return 12;
+ DISPLAY("\nError: not enough memory!\n");
+ fclose(inFile);
+ free(orig_buff);
+ free(compressed_buff);
+ free(chunkP);
+ return(12);
}
- // Fill input buffer
+ /* Fill in src buffer */
DISPLAY("Loading %s... \r", inFileName);
readSize = fread(orig_buff, 1, benchedSize, inFile);
fclose(inFile);
@@ -616,11 +635,11 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
return 13;
}
- // Calculating input Checksum
- crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0);
+ /* Calculating input Checksum */
+ crcOriginal = XXH32(orig_buff, benchedSize,0);
- // Bench
+ /* Bench */
{
int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb;
size_t cSize=0;
@@ -629,67 +648,80 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
DISPLAY("\r%79s\r", "");
DISPLAY(" %s : \n", inFileName);
- // Compression Algorithms
- for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
+ /* Bench Compression Algorithms */
+ for (cAlgNb=0; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (g_compressionTest); cAlgNb++)
{
const char* compressorName;
int (*compressionFunction)(const char*, char*, int);
- void* (*initFunction)(const char*) = NULL;
+ void (*initFunction)(void) = NULL;
double bestTime = 100000000.;
- // Init data chunks
+ /* filter compressionAlgo only */
+ if ((g_compressionAlgo != ALL_COMPRESSORS) && (g_compressionAlgo != cAlgNb)) continue;
+
+ /* Init data chunks */
{
int i;
size_t remaining = benchedSize;
char* in = orig_buff;
char* out = compressed_buff;
- nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize);
+ nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize);
for (i=0; i<nbChunks; i++)
{
chunkP[i].id = i;
- chunkP[i].origBuffer = in; in += chunkSize;
- if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
+ chunkP[i].origBuffer = in; in += g_chunkSize;
+ if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
chunkP[i].compressedSize = 0;
}
}
- if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue;
-
switch(cAlgNb)
{
- case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break;
- case 2 : compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
- case 3 : compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
- case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
- case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break;
- case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break;
- case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break;
- case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
- case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
- case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
- case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break;
- case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
- case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
- case 14: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame";
+ case 0 : DISPLAY("Compression functions : \n"); continue;
+ case 1 : compressionFunction = local_LZ4_compress_default_large; compressorName = "LZ4_compress_default"; break;
+ case 2 : compressionFunction = local_LZ4_compress_default_small; compressorName = "LZ4_compress_default(small dst)"; break;
+ case 3 : compressionFunction = local_LZ4_compress_fast0; compressorName = "LZ4_compress_fast(0)"; break;
+ case 4 : compressionFunction = local_LZ4_compress_fast1; compressorName = "LZ4_compress_fast(1)"; break;
+ case 5 : compressionFunction = local_LZ4_compress_fast2; compressorName = "LZ4_compress_fast(2)"; break;
+ case 6 : compressionFunction = local_LZ4_compress_fast17; compressorName = "LZ4_compress_fast(17)"; break;
+ case 7 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break;
+ case 8 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break;
+
+ case 10: compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break;
+ case 11: compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
+ case 12 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
+ case 13: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
+ case 14: compressionFunction = local_LZ4_compressHC_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_continue"; break;
+ case 15: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
+ case 20: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
+ case 30: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame";
chunkP[0].origSize = (int)benchedSize; nbChunks=1;
break;
- case 15: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict";
- LZ4_loadDict(&LZ4_dict, chunkP[0].origBuffer, chunkP[0].origSize);
+ case 40: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict";
+ LZ4_loadDict(&LZ4_stream, chunkP[0].origBuffer, chunkP[0].origSize);
break;
- case 16: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC";
- LZ4_loadDictHC(&LZ4_dictHC, chunkP[0].origBuffer, chunkP[0].origSize);
+ case 41: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC";
+ LZ4_loadDictHC(&LZ4_streamHC, chunkP[0].origBuffer, chunkP[0].origSize);
break;
- default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
+ case 60: DISPLAY("Obsolete compression functions : \n"); continue;
+ case 61: compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break;
+ case 62: compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
+ case 63: compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
+ case 64: compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
+ case 65: compressionFunction = local_LZ4_compress_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_continue"; break;
+ case 66: compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_limitedOutput_continue"; break;
+ default :
+ continue; /* unknown ID : just skip */
}
- for (loopNb = 1; loopNb <= nbIterations; loopNb++)
+ for (loopNb = 1; loopNb <= g_nbIterations; loopNb++)
{
double averageTime;
int milliTime;
PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize);
- { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warming up memory
+ { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } /* warming up memory */
nb_loops = 0;
milliTime = BMK_GetMilliStart();
@@ -697,17 +729,17 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
milliTime = BMK_GetMilliStart();
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
- if (initFunction!=NULL) ctx = (LZ4_stream_t*)initFunction(chunkP[0].origBuffer);
+ if (initFunction!=NULL) initFunction();
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
{
chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
}
- if (initFunction!=NULL) free(ctx);
nb_loops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
+ nb_loops += !nb_loops; /* avoid division by zero */
averageTime = (double)milliTime / nb_loops;
if (averageTime < bestTime) bestTime = averageTime;
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
@@ -719,24 +751,22 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
else
DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
-
- totalCTime[cAlgNb] += bestTime;
- totalCSize[cAlgNb] += cSize;
}
- // Prepare layout for decompression
- // Init data chunks
+ /* Prepare layout for decompression */
+ /* Init data chunks */
{
int i;
size_t remaining = benchedSize;
char* in = orig_buff;
char* out = compressed_buff;
- nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize);
+
+ nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize);
for (i=0; i<nbChunks; i++)
{
chunkP[i].id = i;
- chunkP[i].origBuffer = in; in += chunkSize;
- if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
+ chunkP[i].origBuffer = in; in += g_chunkSize;
+ if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
chunkP[i].compressedSize = 0;
}
@@ -747,39 +777,45 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
}
- // Decompression Algorithms
- for (dAlgNb=1; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); dAlgNb++)
+ /* Decompression Algorithms */
+ for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (g_decompressionTest); dAlgNb++)
{
- //const char* dName = decompressionNames[dAlgNb];
const char* dName;
int (*decompressionFunction)(const char*, char*, int, int);
double bestTime = 100000000.;
- if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb)) continue;
+ if ((g_decompressionAlgo != ALL_DECOMPRESSORS) && (g_decompressionAlgo != dAlgNb)) continue;
switch(dAlgNb)
{
+ case 0: DISPLAY("Decompression functions : \n"); continue;
case 1: decompressionFunction = local_LZ4_decompress_fast; dName = "LZ4_decompress_fast"; break;
- case 2: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; dName = "LZ4_decompress_fast_withPrefix64k"; break;
case 3: decompressionFunction = local_LZ4_decompress_fast_usingDict; dName = "LZ4_decompress_fast_usingDict"; break;
case 4: decompressionFunction = LZ4_decompress_safe; dName = "LZ4_decompress_safe"; break;
- case 5: decompressionFunction = LZ4_decompress_safe_withPrefix64k; dName = "LZ4_decompress_safe_withPrefix64k"; break;
case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break;
case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; break;
case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress";
errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL);
- if (LZ4F_isError(errorCode)) { DISPLAY("Preparation error compressing frame\n"); return 1; }
+ if (LZ4F_isError(errorCode))
+ {
+ DISPLAY("Error while preparing compressed frame\n");
+ free(orig_buff);
+ free(compressed_buff);
+ free(chunkP);
+ return 1;
+ }
chunkP[0].origSize = (int)benchedSize;
chunkP[0].compressedSize = (int)errorCode;
nbChunks = 1;
break;
- default : DISPLAY("ERROR ! Bad decompression algorithm Id !! \n"); free(chunkP); return 1;
+ default :
+ continue; /* skip if unknown ID */
}
- { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking
+ { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing source area, for CRC checking */
- for (loopNb = 1; loopNb <= nbIterations; loopNb++)
+ for (loopNb = 1; loopNb <= g_nbIterations; loopNb++)
{
double averageTime;
int milliTime;
@@ -802,6 +838,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
}
milliTime = BMK_GetMilliSpan(milliTime);
+ nb_loops += !nb_loops; /* Avoid division by zero */
averageTime = (double)milliTime / nb_loops;
if (averageTime < bestTime) bestTime = averageTime;
@@ -813,24 +850,21 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
}
DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
-
- totalDTime[dAlgNb] += bestTime;
}
-
}
-
free(orig_buff);
free(compressed_buff);
free(chunkP);
}
- if (BMK_pause) { printf("press enter...\n"); getchar(); }
+ LZ4F_freeDecompressionContext(g_dCtx);
+ if (g_pause) { printf("press enter...\n"); (void)getchar(); }
return 0;
}
-int usage(char* exename)
+static int usage(char* exename)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
@@ -841,7 +875,7 @@ int usage(char* exename)
return 0;
}
-int usage_advanced(void)
+static int usage_advanced(void)
{
DISPLAY( "\nAdvanced options :\n");
DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS);
@@ -851,7 +885,7 @@ int usage_advanced(void)
return 0;
}
-int badusage(char* exename)
+static int badusage(char* exename)
{
DISPLAY("Wrong parameters\n");
usage(exename);
@@ -877,7 +911,7 @@ int main(int argc, char** argv)
if(!argument) continue; // Protection if argument empty
if (!strcmp(argument, "--no-prompt"))
{
- no_prompt = 1;
+ g_noPrompt = 1;
continue;
}
@@ -892,22 +926,22 @@ int main(int argc, char** argv)
{
// Select compression algorithm only
case 'c':
- decompressionTest = 0;
+ g_decompressionTest = 0;
while ((argument[1]>= '0') && (argument[1]<= '9'))
{
- compressionAlgo *= 10;
- compressionAlgo += argument[1] - '0';
+ g_compressionAlgo *= 10;
+ g_compressionAlgo += argument[1] - '0';
argument++;
}
break;
// Select decompression algorithm only
case 'd':
- compressionTest = 0;
+ g_compressionTest = 0;
while ((argument[1]>= '0') && (argument[1]<= '9'))
{
- decompressionAlgo *= 10;
- decompressionAlgo += argument[1] - '0';
+ g_decompressionAlgo *= 10;
+ g_decompressionAlgo += argument[1] - '0';
argument++;
}
break;
@@ -928,7 +962,7 @@ int main(int argc, char** argv)
{
int B = argument[1] - '0';
int S = 1 << (8 + 2*B);
- BMK_SetBlocksize(S);
+ BMK_setBlocksize(S);
argument++;
break;
}
@@ -940,16 +974,16 @@ _exit_blockProperties:
// Modify Nb Iterations
case 'i':
- if ((argument[1] >='1') && (argument[1] <='9'))
+ if ((argument[1] >='0') && (argument[1] <='9'))
{
int iters = argument[1] - '0';
- BMK_SetNbIterations(iters);
+ BMK_setNbIterations(iters);
argument++;
}
break;
// Pause at the end (hidden option)
- case 'p': BMK_SetPause(); break;
+ case 'p': BMK_setPause(); break;
// Unknown command
default : badusage(exename); return 1;
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index 3d3cf8e..6588290 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -19,7 +19,6 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4
- LZ4 source mirror : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
@@ -154,10 +153,10 @@ static U32 FUZ_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
- rand32 += PRIME2;
+ rand32 ^= PRIME2;
rand32 = FUZ_rotl32(rand32, 13);
*src = rand32;
- return rand32 >> 3;
+ return rand32;
}
@@ -169,27 +168,28 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou
size_t pos = 0;
U32 P32 = (U32)(32768 * proba);
- // First Byte
- BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
+ /* First Bytes */
+ while (pos < 20)
+ BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
while (pos < bufferSize)
{
- // Select : Literal (noise) or copy (within 64K)
+ /* Select : Literal (noise) or copy (within 64K) */
if (FUZ_RAND15BITS < P32)
{
- // Copy (within 64K)
+ /* Copy (within 64K) */
size_t match, d;
size_t length = FUZ_RANDLENGTH + 4;
size_t offset = FUZ_RAND15BITS + 1;
- if (offset > pos) offset = pos;
+ while (offset > pos) offset >>= 1;
d = pos + length;
- if (d > bufferSize) d = bufferSize;
+ while (d > bufferSize) d = bufferSize;
match = pos - offset;
while (pos < d) BBuffer[pos++] = BBuffer[match++];
}
else
{
- // Literal (noise)
+ /* Literal (noise) */
size_t d;
size_t length = FUZ_RANDLENGTH;
d = pos + length;
@@ -210,7 +210,7 @@ static int FUZ_AddressOverflow(void)
printf("Overflow tests : ");
- // Only possible in 32-bits
+ /* Only possible in 32-bits */
if (sizeof(void*)==8)
{
printf("64 bits mode : no overflow \n");
@@ -300,7 +300,17 @@ static void FUZ_displayUpdate(unsigned testNb)
}
-static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const double compressibility)
+static void FUZ_findDiff(const void* buff1, const void* buff2)
+{
+ const BYTE* b1 = (const BYTE*)buff1;
+ const BYTE* b2 = (const BYTE*)buff2;
+ size_t i=0;
+ while (b1[i]==b2[i]) i++;
+ DISPLAY("Wrong Byte at position %u\n", (unsigned)i);
+}
+
+
+static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration)
{
unsigned long long bytes = 0;
unsigned long long cbytes = 0;
@@ -313,34 +323,35 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
int ret;
unsigned cycleNb;
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
- printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
+ printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); }
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
- void* LZ4continue;
LZ4_stream_t LZ4dict;
LZ4_streamHC_t LZ4dictHC;
U32 crcOrig, crcCheck;
U32 coreRandState = seed;
U32 randState = coreRandState ^ PRIME3;
int result = 0;
+ const U32 startTime = FUZ_GetMilliStart();
- // init
+ /* init */
memset(&LZ4dict, 0, sizeof(LZ4dict));
+ duration *= 1000;
- // Create compressible test buffer
+ /* Create compressible test buffer */
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
compressedBuffer = (char*)malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
- // move to startCycle
+ /* move to startCycle */
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
{
(void)FUZ_rand(&coreRandState);
- if (0) // some problems related to dictionary re-use; in this case, enable this loop
+ if (0) /* some problems are related to dictionary re-use; in this case, enable this loop */
{
int dictSize, blockSize, blockStart;
char* dict;
@@ -362,8 +373,8 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
}
}
- // Test loop
- for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
+ /* Main test loop */
+ for (cycleNb = startCycle; (cycleNb < nbCycles) || (FUZ_GetMilliSpan(startTime) < duration) ; cycleNb++)
{
U32 testNb = 0;
char* dict;
@@ -375,7 +386,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
(void)FUZ_rand(&coreRandState);
randState = coreRandState ^ PRIME3;
- // Select block to test
+ /* Select block to test */
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
@@ -385,23 +396,58 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
/* Compression tests */
- // Test compression HC
+ /* Test compression destSize */
+ FUZ_DISPLAYTEST;
+ {
+ int srcSize = blockSize;
+ int targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
+ char endCheck = FUZ_rand(&randState) & 255;
+ compressedBuffer[targetSize] = endCheck;
+ ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize);
+ FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !");
+ FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !");
+ FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !");
+ DISPLAY("destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize);
+ if (targetSize>0)
+ {
+ FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed");
+ /* check correctness */
+ FUZ_DISPLAYTEST;
+
+ crcOrig = XXH32(block, srcSize, 0);
+ compressedSize = ret;
+ endCheck = FUZ_rand(&randState) & 255;
+ decodedBuffer[srcSize] = endCheck;
+ ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
+ FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize");
+ FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
+ FUZ_CHECKTEST(decodedBuffer[srcSize] != endCheck, "LZ4_decompress_safe() overwrite dst buffer !");
+ crcCheck = XXH32(decodedBuffer, srcSize, 0);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe() corrupted decoded data");
+
+ DISPLAY(" OK \n");
+ }
+ else
+ DISPLAY(" \n");
+ }
+
+ /* Test compression HC */
FUZ_DISPLAYTEST;
ret = LZ4_compressHC(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
HCcompressedSize = ret;
- // Test compression HC using external state
+ /* Test compression HC using external state */
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
- // Test compression using external state
+ /* Test compression using external state */
FUZ_DISPLAYTEST;
ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
- // Test compression
+ /* Test compression */
FUZ_DISPLAYTEST;
ret = LZ4_compress(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
@@ -411,7 +457,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
crcOrig = XXH32(block, blockSize, 0);
- // Test decoding with output size being exactly what's necessary => must work
+ /* Test decoding with output size being exactly what's necessary => must work */
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
@@ -419,19 +465,19 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
- // Test decoding with one byte missing => must fail
+ /* Test decoding with one byte missing => must fail */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
- // Test decoding with one byte too much => must fail
+ /* Test decoding with one byte too much => must fail */
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
- // Test decoding with output size exactly what's necessary => must work
+ /* Test decoding with output size exactly what's necessary => must work */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
@@ -545,16 +591,18 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
/* Compress using dictionary */
FUZ_DISPLAYTEST;
- LZ4continue = LZ4_create (dict);
- LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
- blockContinueCompressedSize = LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, block, compressedBuffer, blockSize);
- FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
- free (LZ4continue);
+ {
+ LZ4_stream_t LZ4_stream;
+ LZ4_resetStream(&LZ4_stream);
+ LZ4_compress_continue (&LZ4_stream, dict, compressedBuffer, dictSize); /* Just to fill hash tables */
+ blockContinueCompressedSize = LZ4_compress_continue (&LZ4_stream, block, compressedBuffer, blockSize);
+ FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
+ }
/* Decompress with dictionary as prefix */
FUZ_DISPLAYTEST;
memcpy(decodedBuffer, dict, dictSize);
- ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize);
+ ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
if (crcCheck!=crcOrig)
@@ -567,14 +615,14 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
- ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize);
- FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data");
+ ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
+ FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
- // Compress using External dictionary
+ /* Compress using External dictionary */
FUZ_DISPLAYTEST;
- dict -= (FUZ_rand(&randState) & 0xF) + 1; // Separation, so it is an ExtDict
+ dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
@@ -583,7 +631,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
- FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
+ FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
@@ -591,7 +639,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
- // Decompress with dictionary as external
+ /* Decompress with dictionary as external */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
@@ -599,11 +647,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
- {
- int i=0;
- while (block[i]==decodedBuffer[i]) i++;
- printf("Wrong Byte at position %i/%i\n", i, blockSize);
- }
+ FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
@@ -617,7 +661,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)");
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
@@ -638,9 +682,9 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
}
}
- // Compress HC using External dictionary
+ /* Compress HC using External dictionary */
FUZ_DISPLAYTEST;
- dict -= (FUZ_rand(&randState) & 7); // even bigger separation
+ dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7);
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
@@ -665,29 +709,26 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
- {
- int i=0;
- while (block[i]==decodedBuffer[i]) i++;
- printf("Wrong Byte at position %i/%i\n", i, blockSize);
- }
+ FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
-
- // ***** End of tests *** //
- // Fill stats
+ /* ***** End of tests *** */
+ /* Fill stats */
bytes += blockSize;
cbytes += compressedSize;
hcbytes += HCcompressedSize;
ccbytes += blockContinueCompressedSize;
}
+ if (nbCycles<=1) nbCycles = cycleNb; /* end by time */
+ bytes += !bytes; /* avoid division by 0 */
printf("\r%7u /%7u - ", cycleNb, nbCycles);
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100);
- // unalloc
+ /* release memory */
{
_exit:
free(CNBuffer);
@@ -719,13 +760,13 @@ static void FUZ_unitTests(void)
char ringBuffer[ringBufferSize];
U32 randState = 1;
- // Init
+ /* Init */
FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
- // 32-bits address space overflow test
+ /* 32-bits address space overflow test */
FUZ_AddressOverflow();
- // LZ4 streaming tests
+ /* LZ4 streaming tests */
{
LZ4_stream_t* statePtr;
LZ4_stream_t streamingState;
@@ -733,12 +774,12 @@ static void FUZ_unitTests(void)
U64 crcNew;
int result;
- // Allocation test
+ /* Allocation test */
statePtr = LZ4_createStream();
FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
LZ4_freeStream(statePtr);
- // simple compression test
+ /* simple compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStream(&streamingState);
result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
@@ -749,7 +790,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
- // ring buffer test
+ /* ring buffer test */
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
@@ -783,7 +824,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
- // prepare next message
+ /* prepare next message */
iNext += messageSize;
rNext += messageSize;
dNext += messageSize;
@@ -794,21 +835,20 @@ static void FUZ_unitTests(void)
}
}
- // LZ4 HC streaming tests
+ /* LZ4 HC streaming tests */
{
LZ4_streamHC_t* sp;
LZ4_streamHC_t sHC;
- //XXH64_state_t xxh;
U64 crcOrig;
U64 crcNew;
int result;
- // Allocation test
+ /* Allocation test */
sp = LZ4_createStreamHC();
FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed");
LZ4_freeStreamHC(sp);
- // simple compression test
+ /* simple HC compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
@@ -819,7 +859,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
- // simple dictionary compression test
+ /* simple dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 64 KB);
@@ -831,7 +871,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption");
- // multiple HC compression test with dictionary
+ /* multiple HC compression test with dictionary */
{
int result1, result2;
int segSize = testCompressedSize / 2;
@@ -851,7 +891,7 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
}
- // remote dictionary HC compression test
+ /* remote dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 32 KB);
@@ -863,7 +903,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption");
- // multiple HC compression with ext. dictionary
+ /* multiple HC compression with ext. dictionary */
{
XXH64_state_t crcOrigState;
XXH64_state_t crcNewState;
@@ -912,7 +952,7 @@ static void FUZ_unitTests(void)
}
}
- // ring buffer test
+ /* ring buffer test */
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
@@ -956,24 +996,51 @@ static void FUZ_unitTests(void)
}
}
- // small decoder-side ring buffer test
+ /* small decoder-side ring buffer test */
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
LZ4_streamDecode_t decodeState;
- const U32 maxMessageSizeLog = 10;
+ const U32 maxMessageSizeLog = 12;
const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
- U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
+ U32 messageSize;
U32 totalMessageSize = 0;
U32 iNext = 0;
U32 dNext = 0;
- const U32 dBufferSize = 64 KB + maxMessageSizeMask;
+ const U32 dBufferSize = 64 KB;
XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNew, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_setStreamDecode(&decodeState, NULL, 0);
+#define BSIZE1 65537
+#define BSIZE2 16435
+
+ /* first block */
+
+ messageSize = BSIZE1;
+ XXH64_update(&xxhOrig, testInput + iNext, messageSize);
+ crcOrig = XXH64_digest(&xxhOrig);
+
+ result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+ FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
+
+ result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
+ FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
+
+ XXH64_update(&xxhNew, testVerify + dNext, messageSize);
+ crcNew = XXH64_digest(&xxhNew);
+ FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
+
+ /* prepare next message */
+ dNext += messageSize;
+ totalMessageSize += messageSize;
+ messageSize = BSIZE2;
+ iNext = 132000;
+ memcpy(testInput + iNext, testInput + 8, messageSize);
+ if (dNext > dBufferSize) dNext = 0;
+
while (totalMessageSize < 9 MB)
{
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
@@ -983,18 +1050,20 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
- FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
+ FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = XXH64_digest(&xxhNew);
- FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
+ if (crcOrig != crcNew)
+ FUZ_findDiff(testInput + iNext, testVerify + dNext);
+ FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test");
/* prepare next message */
dNext += messageSize;
totalMessageSize += messageSize;
messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
iNext = (FUZ_rand(&randState) & 65535);
- if (dNext + messageSize > dBufferSize) dNext = 0;
+ if (dNext > dBufferSize) dNext = 0;
}
}
}
@@ -1013,6 +1082,7 @@ static int FUZ_usage(char* programName)
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS);
+ DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
DISPLAY( " -s# : Select seed (default:prompt user)\n");
DISPLAY( " -t# : Select starting test number (default:0)\n");
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
@@ -1033,8 +1103,9 @@ int main(int argc, char** argv)
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int pause = 0;
char* programName = argv[0];
+ U32 duration = 0;
- // Check command line
+ /* Check command line */
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
@@ -1066,7 +1137,7 @@ int main(int argc, char** argv)
case 'i':
argument++;
- nbTests=0;
+ nbTests = 0; duration = 0;
while ((*argument>='0') && (*argument<='9'))
{
nbTests *= 10;
@@ -1075,6 +1146,31 @@ int main(int argc, char** argv)
}
break;
+ case 'T':
+ argument++;
+ nbTests = 0; duration = 0;
+ for (;;)
+ {
+ switch(*argument)
+ {
+ case 'm': duration *= 60; argument++; continue;
+ case 's':
+ case 'n': argument++; continue;
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9': duration *= 10; duration += *argument++ - '0'; continue;
+ }
+ break;
+ }
+ break;
+
case 's':
argument++;
seed=0; seedset=1;
@@ -1115,7 +1211,6 @@ int main(int argc, char** argv)
}
}
- // Get Seed
printf("Starting LZ4 fuzzer (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION);
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
@@ -1127,11 +1222,11 @@ int main(int argc, char** argv)
if (nbTests<=0) nbTests=1;
{
- int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100);
+ int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration);
if (pause)
{
DISPLAY("press enter ... \n");
- getchar();
+ (void)getchar();
}
return result;
}
diff --git a/programs/lz4.1 b/programs/lz4.1
index 1225832..a79fed3 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -158,13 +158,13 @@ for files that have not been compressed with
force write to standard output, even if it is the console
.TP
-.BR \-m
+.BR \-m ", " \--multiple
Multiple file names.
By default, the second filename is used as the output filename for the compressed file.
With
.B -m
-, you can specify any number of input filenames, each of them will be compressed
-with the resulting compressed file named
+, you can specify any number of input filenames. Each of them will be compressed
+independently, and the resulting name of the compressed file will be
.B filename.lz4
.
@@ -177,15 +177,15 @@ with the resulting compressed file named
block dependency (improve compression ratio)
.TP
.B \--[no-]frame-crc
- disable stream checksum (default:enabled)
+ select frame checksum (default:enabled)
.TP
.B \--[no-]content-size
- compressed file includes original size (default:not present)
+ header includes original size (default:not present)
Note : this option can only be activated when the original size can be determined,
-hence for a file. It won't work with unknown source size, such as stdin pipe.
+hence for a file. It won't work with unknown source size, such as stdin or pipe.
.TP
.B \--[no-]sparse
- enable sparse file (default:disabled)(experimental)
+ sparse file support (default:enabled)
.TP
.B \-l
use Legacy format (useful for Linux Kernel compression)
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index da5da71..970856d 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -65,7 +65,7 @@
/****************************
* OS-specific Includes
*****************************/
-#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
# include <io.h> /* _isatty */
# ifdef __MINGW32__
int _fileno(FILE *stream); /* MINGW somehow forgets to include this prototype into <stdio.h> */
@@ -102,7 +102,7 @@
***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-static unsigned displayLevel = 2; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
+static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */
/**************************************
@@ -175,7 +175,7 @@ static int usage_advanced(void)
/* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */
DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n");
DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
- DISPLAY( "--sparse : enable sparse file (default:disabled)(experimental)\n");
+ DISPLAY( "--[no-]sparse : sparse file support (default:enabled)\n");
DISPLAY( "Benchmark arguments :\n");
DISPLAY( " -b : benchmark file(s)\n");
DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");
@@ -251,7 +251,7 @@ static int badusage(void)
static void waitEnter(void)
{
DISPLAY("Press enter to continue...\n");
- getchar();
+ (void)getchar();
}
@@ -265,7 +265,8 @@ int main(int argc, char** argv)
forceStdout=0,
forceCompress=0,
main_pause=0,
- multiple_inputs=0;
+ multiple_inputs=0,
+ operationResult=0;
const char* input_filename=0;
const char* output_filename=0;
char* dynNameSpace=0;
@@ -294,12 +295,13 @@ int main(int argc, char** argv)
/* long commands (--long-word) */
if (!strcmp(argument, "--compress")) { forceCompress = 1; continue; }
if ((!strcmp(argument, "--decompress"))
- || (!strcmp(argument, "--uncompress"))) { decode = 1; continue; }
+ || (!strcmp(argument, "--uncompress"))) { decode = 1; continue; }
+ if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; if (inFileNames==NULL) inFileNames = (const char**)malloc(argc * sizeof(char*)); continue; }
if (!strcmp(argument, "--test")) { decode = 1; LZ4IO_setOverwrite(1); output_filename=nulmark; continue; }
if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; }
if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; }
if ((!strcmp(argument, "--stdout"))
- || (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; }
+ || (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; }
if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; }
if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; }
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; }
@@ -311,6 +313,7 @@ int main(int argc, char** argv)
if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
if (!strcmp(argument, "--keep")) { continue; } /* keep source file (default anyway; just for xz/lzma compatibility) */
+
/* Short commands (note : aggregated short commands are allowed) */
if (argument[0]=='-')
{
@@ -420,11 +423,15 @@ int main(int argc, char** argv)
/* Modify Nb Iterations (benchmark only) */
case 'i':
- if ((argument[1] >='1') && (argument[1] <='9'))
{
- int iters = argument[1] - '0';
+ unsigned iters = 0;
+ while ((argument[1] >='0') && (argument[1] <='9'))
+ {
+ iters *= 10;
+ iters += argument[1] - '0';
+ argument++;
+ }
BMK_setNbIterations(iters);
- argument++;
}
break;
@@ -463,14 +470,19 @@ int main(int argc, char** argv)
if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10);
/* No input filename ==> use stdin */
- if (multiple_inputs) input_filename = inFileNames[0], output_filename = (char*)(inFileNames[0]);
+ if (multiple_inputs) input_filename = inFileNames[0], output_filename = (const char*)(inFileNames[0]);
if(!input_filename) { input_filename=stdinmark; }
/* Check if input or output are defined as console; trigger an error in this case */
if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
/* Check if benchmark is selected */
- if (bench) return BMK_benchFiles(inFileNames, ifnIdx, cLevel);
+ if (bench)
+ {
+ int bmkResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel);
+ free((void*)inFileNames);
+ return bmkResult;
+ }
/* No output filename ==> try to select one automatically (when possible) */
while (!output_filename)
@@ -509,32 +521,40 @@ int main(int argc, char** argv)
/* Check if output is defined as console; trigger an error in this case */
if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage();
- /* No warning message in pure pipe mode (stdin + stdout) */
+ /* Downgrade notification level in pure pipe mode (stdin + stdout) and multiple file mode */
if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;
+ if ((multiple_inputs) && (displayLevel==2)) displayLevel=1;
/* IO Stream/File */
LZ4IO_setNotificationLevel(displayLevel);
- if (decode) DEFAULT_DECOMPRESSOR(input_filename, output_filename);
+ if (decode)
+ {
+ if (multiple_inputs)
+ operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION);
+ else
+ DEFAULT_DECOMPRESSOR(input_filename, output_filename);
+ }
else
{
- /* compression is default action */
- if (legacy_format)
- {
- DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
- LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
- }
+ /* compression is default action */
+ if (legacy_format)
+ {
+ DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
+ LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
+ }
+ else
+ {
+ if (multiple_inputs)
+ operationResult = LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel);
else
- {
- if (multiple_inputs)
- LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel);
- else
- DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
- }
+ DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
+ }
}
if (main_pause) waitEnter();
free(dynNameSpace);
free((void*)inFileNames);
+ if (operationResult != 0) return operationResult;
return 0;
}
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 02e03c8..209f5ed 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -46,22 +46,22 @@
/*****************************
* Includes
*****************************/
-#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */
+#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <time.h> /* clock */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
#include "lz4io.h"
-#include "lz4.h" /* still required for legacy format */
-#include "lz4hc.h" /* still required for legacy format */
+#include "lz4.h" /* still required for legacy format */
+#include "lz4hc.h" /* still required for legacy format */
#include "lz4frame.h"
/******************************
* OS-specific Includes
******************************/
-#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _fileno, _get_osfhandle */
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
@@ -131,7 +131,7 @@ static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT;
static int g_blockChecksum = 0;
static int g_streamChecksum = 1;
static int g_blockIndependence = 1;
-static int g_sparseFileSupport = 0;
+static int g_sparseFileSupport = 1;
static int g_contentSizeFlag = 0;
static const int minBlockSizeID = 4;
@@ -159,7 +159,7 @@ static const int maxBlockSizeID = 7;
#define EXTENDED_ARGUMENTS
#define EXTENDED_HELP
#define EXTENDED_FORMAT
-#define DEFAULT_DECOMPRESSOR decodeLZ4S
+#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F
/* ************************************************** */
@@ -240,7 +240,7 @@ static unsigned long long LZ4IO_GetFileSize(const char* infilename)
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
- if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* failure, or is not a regular file */
return (unsigned long long)statbuf.st_size;
}
@@ -253,7 +253,7 @@ static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))
static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; }
-static int get_fileHandle(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput)
+static int LZ4IO_getFiles(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput)
{
if (!strcmp (input_filename, stdinmark))
@@ -267,6 +267,12 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam
*pfinput = fopen(input_filename, "rb");
}
+ if ( *pfinput==0 )
+ {
+ DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", input_filename);
+ return 1;
+ }
+
if (!strcmp (output_filename, stdoutmark))
{
DISPLAYLEVEL(4,"Using stdout for output\n");
@@ -283,18 +289,18 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam
fclose(*pfoutput);
if (!g_overwrite)
{
- char ch;
+ int ch = 'Y';
DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
- DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
- if (g_displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */
- ch = (char)getchar();
- if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
+ if ((g_displayLevel <= 1) || (*pfinput == stdin))
+ EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */
+ DISPLAYLEVEL(2, "Overwrite ? (Y/n) : ");
+ while((ch = getchar()) != '\n' && ch != EOF) /* flush integrated */
+ if ((ch!='Y') && (ch!='y')) EXM_THROW(12, "No. Operation aborted : %s already exists", output_filename);
}
}
*pfoutput = fopen( output_filename, "wb" );
}
- if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);
if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
return 0;
@@ -316,16 +322,23 @@ static void LZ4IO_writeLE32 (void* p, unsigned value32)
dstPtr[3] = (unsigned char)(value32 >> 24);
}
+static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSize, int cLevel)
+{
+ (void)cLevel;
+ return LZ4_compress_fast(src, dst, srcSize, dstSize, 1);
+}
+
/* LZ4IO_compressFilename_Legacy :
* This function is intentionally "hidden" (not published in .h)
* It generates compressed streams using the old 'legacy' format */
int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel)
{
- int (*compressionFunction)(const char*, char*, int);
+ int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
unsigned long long filesize = 0;
unsigned long long compressedfilesize = MAGICNUMBER_SIZE;
char* in_buff;
char* out_buff;
+ const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE);
FILE* finput;
FILE* foutput;
clock_t start, end;
@@ -334,14 +347,14 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
/* Init */
start = clock();
- if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC;
+ if (compressionlevel < 3) compressionFunction = LZ4IO_LZ4_compress; else compressionFunction = LZ4_compress_HC;
- get_fileHandle(input_filename, output_filename, &finput, &foutput);
- if ((g_displayLevel==2) && (compressionlevel==1)) g_displayLevel=3;
+ if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput))
+ EXM_THROW(20, "File error");
/* Allocate Memory */
in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
- out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
+ out_buff = (char*)malloc(outBuffSize);
if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
/* Write Archive Header */
@@ -359,9 +372,9 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
filesize += inSize;
/* Compress Block */
- outSize = compressionFunction(in_buff, out_buff+4, inSize);
+ outSize = compressionFunction(in_buff, out_buff+4, inSize, outBuffSize, compressionlevel);
compressedfilesize += outSize+4;
- DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
+ DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
LZ4IO_writeLE32(out_buff, outSize);
@@ -372,6 +385,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
/* Status */
end = clock();
DISPLAYLEVEL(2, "\r%79s\r", "");
+ filesize += !filesize; /* avoid divide by zero */
DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
(unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
{
@@ -393,128 +407,210 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
* Compression using Frame format
*********************************************/
-int LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionLevel)
+typedef struct {
+ void* srcBuffer;
+ size_t srcBufferSize;
+ void* dstBuffer;
+ size_t dstBufferSize;
+ LZ4F_compressionContext_t ctx;
+} cRess_t;
+
+static cRess_t LZ4IO_createCResources(void)
+{
+ const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId);
+ cRess_t ress;
+ LZ4F_errorCode_t errorCode;
+
+ errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION);
+ if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
+
+ /* Allocate Memory */
+ ress.srcBuffer = malloc(blockSize);
+ ress.srcBufferSize = blockSize;
+ ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */
+ ress.dstBuffer = malloc(ress.dstBufferSize);
+ if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
+
+ return ress;
+}
+
+static void LZ4IO_freeCResources(cRess_t ress)
+{
+ LZ4F_errorCode_t errorCode;
+ free(ress.srcBuffer);
+ free(ress.dstBuffer);
+ errorCode = LZ4F_freeCompressionContext(ress.ctx);
+ if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
+}
+
+/*
+ * LZ4IO_compressFilename_extRess()
+ * result : 0 : compression completed correctly
+ * 1 : missing or pb opening srcFileName
+ */
+static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel)
{
unsigned long long filesize = 0;
unsigned long long compressedfilesize = 0;
- char* in_buff;
- char* out_buff;
- FILE* finput;
- FILE* foutput;
- clock_t start, end;
- int blockSize;
- size_t sizeCheck, headerSize, readSize, outBuffSize;
- LZ4F_compressionContext_t ctx;
- LZ4F_errorCode_t errorCode;
+ FILE* srcFile;
+ FILE* dstFile;
+ void* const srcBuffer = ress.srcBuffer;
+ void* const dstBuffer = ress.dstBuffer;
+ const size_t dstBufferSize = ress.dstBufferSize;
+ const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId);
+ size_t sizeCheck, headerSize, readSize;
+ LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */
LZ4F_preferences_t prefs;
/* Init */
- start = clock();
memset(&prefs, 0, sizeof(prefs));
- if ((g_displayLevel==2) && (compressionLevel>=3)) g_displayLevel=3;
- errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
- get_fileHandle(input_filename, output_filename, &finput, &foutput);
- blockSize = LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId);
+
+ /* File check */
+ if (LZ4IO_getFiles(srcFileName, dstFileName, &srcFile, &dstFile)) return 1;
/* Set compression parameters */
prefs.autoFlush = 1;
prefs.compressionLevel = compressionLevel;
- prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence;
- prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId;
- prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum;
+ prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence;
+ prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId;
+ prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum;
if (g_contentSizeFlag)
{
- unsigned long long fileSize = LZ4IO_GetFileSize(input_filename);
+ unsigned long long fileSize = LZ4IO_GetFileSize(srcFileName);
prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */
+ if (fileSize==0)
+ DISPLAYLEVEL(3, "Warning : cannot determine uncompressed frame content size \n");
}
- /* Allocate Memory */
- in_buff = (char*)malloc(blockSize);
- outBuffSize = LZ4F_compressBound(blockSize, &prefs);
- out_buff = (char*)malloc(outBuffSize);
- if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");
-
- /* Write Archive Header */
- headerSize = LZ4F_compressBegin(ctx, out_buff, outBuffSize, &prefs);
- if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
- sizeCheck = fwrite(out_buff, 1, headerSize, foutput);
- if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header");
- compressedfilesize += headerSize;
-
/* read first block */
- readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
+ readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile);
filesize += readSize;
- /* Main Loop */
- while (readSize>0)
+ /* single-block file */
+ if (readSize < blockSize)
{
- size_t outSize;
-
- /* Compress Block */
- outSize = LZ4F_compressUpdate(ctx, out_buff, outBuffSize, in_buff, readSize, NULL);
- if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize));
- compressedfilesize += outSize;
- DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
+ /* Compress in single pass */
+ size_t cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs);
+ if (LZ4F_isError(cSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(cSize));
+ compressedfilesize += cSize;
+ DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
+ (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */
/* Write Block */
- sizeCheck = fwrite(out_buff, 1, outSize, foutput);
- if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block");
-
- /* Read next block */
- readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
- filesize += readSize;
+ sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile);
+ if (sizeCheck!=cSize) EXM_THROW(35, "Write error : cannot write compressed block");
}
- /* End of Stream mark */
- headerSize = LZ4F_compressEnd(ctx, out_buff, outBuffSize, NULL);
- if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
+ else
- sizeCheck = fwrite(out_buff, 1, headerSize, foutput);
- if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream");
- compressedfilesize += headerSize;
+ /* multiple-blocks file */
+ {
+ /* Write Archive Header */
+ headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs);
+ if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
+ sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
+ if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header");
+ compressedfilesize += headerSize;
+
+ /* Main Loop */
+ while (readSize>0)
+ {
+ size_t outSize;
- /* Close & Free */
- free(in_buff);
- free(out_buff);
- fclose(finput);
- fclose(foutput);
- errorCode = LZ4F_freeCompressionContext(ctx);
- if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
+ /* Compress Block */
+ outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
+ if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize));
+ compressedfilesize += outSize;
+ DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
+
+ /* Write Block */
+ sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile);
+ if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block");
+
+ /* Read next block */
+ readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile);
+ filesize += readSize;
+ }
+
+ /* End of Stream mark */
+ headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
+ if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
+
+ sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
+ if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream");
+ compressedfilesize += headerSize;
+ }
+
+ /* Release files */
+ fclose (srcFile);
+ fclose (dstFile);
/* Final Status */
- end = clock();
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
- (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
+ filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */
+
+ return 0;
+}
+
+
+int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel)
+{
+ clock_t start, end;
+ cRess_t ress;
+ int issueWithSrcFile = 0;
+
+ /* Init */
+ start = clock();
+ ress = LZ4IO_createCResources();
+
+ /* Compress File */
+ issueWithSrcFile += LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel);
+
+ /* Free resources */
+ LZ4IO_freeCResources(ress);
+
+ /* Final Status */
+ end = clock();
{
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;
- DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
+ double seconds = (double)(end - start) / CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
}
- return 0;
+ return issueWithSrcFile;
}
#define FNSPACE 30
-int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel)
+int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel)
{
int i;
- char* outFileName = (char*)malloc(FNSPACE);
+ int missed_files = 0;
+ char* dstFileName = (char*)malloc(FNSPACE);
size_t ofnSize = FNSPACE;
const size_t suffixSize = strlen(suffix);
+ cRess_t ress;
+
+ /* init */
+ ress = LZ4IO_createCResources();
+ /* loop on each file */
for (i=0; i<ifntSize; i++)
{
size_t ifnSize = strlen(inFileNamesTable[i]);
- if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); }
- strcpy(outFileName, inFileNamesTable[i]);
- strcat(outFileName, suffix);
- LZ4IO_compressFilename(inFileNamesTable[i], outFileName, compressionlevel);
+ if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); }
+ strcpy(dstFileName, inFileNamesTable[i]);
+ strcat(dstFileName, suffix);
+
+ missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel);
}
- free(outFileName);
- return 0;
+
+ /* Close & Free */
+ LZ4IO_freeCResources(ress);
+ free(dstFileName);
+
+ return missed_files;
}
@@ -528,15 +624,105 @@ static unsigned LZ4IO_readLE32 (const void* s)
unsigned value32 = srcPtr[0];
value32 += (srcPtr[1]<<8);
value32 += (srcPtr[2]<<16);
- value32 += (srcPtr[3]<<24);
+ value32 += ((unsigned)srcPtr[3])<<24;
return value32;
}
-static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
+static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
+{
+ const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
+ const size_t* ptrT = bufferT;
+ size_t bufferSizeT = bufferSize / sizeT;
+ const size_t* const bufferTEnd = bufferT + bufferSizeT;
+ static const size_t segmentSizeT = (32 KB) / sizeT;
+
+ if (!g_sparseFileSupport) /* normal write */
+ {
+ size_t sizeCheck = fwrite(buffer, 1, bufferSize, file);
+ if (sizeCheck != bufferSize) EXM_THROW(68, "Write error : cannot write decoded block");
+ return 0;
+ }
+
+ /* avoid int overflow */
+ if (storedSkips > 1 GB)
+ {
+ int seekResult = fseek(file, 1 GB, SEEK_CUR);
+ if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file support)");
+ storedSkips -= 1 GB;
+ }
+
+ while (ptrT < bufferTEnd)
+ {
+ size_t seg0SizeT = segmentSizeT;
+ size_t nb0T;
+ int seekResult;
+
+ /* count leading zeros */
+ if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT;
+ bufferSizeT -= seg0SizeT;
+ for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ;
+ storedSkips += (unsigned)(nb0T * sizeT);
+
+ if (nb0T != seg0SizeT) /* not all 0s */
+ {
+ size_t sizeCheck;
+ seekResult = fseek(file, storedSkips, SEEK_CUR);
+ if (seekResult) EXM_THROW(68, "Skip error (sparse file)");
+ storedSkips = 0;
+ seg0SizeT -= nb0T;
+ ptrT += nb0T;
+ sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file);
+ if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block");
+ }
+ ptrT += seg0SizeT;
+ }
+
+ if (bufferSize & maskT) /* size not multiple of sizeT : implies end of block */
+ {
+ const char* const restStart = (const char*)bufferTEnd;
+ const char* restPtr = restStart;
+ size_t restSize = bufferSize & maskT;
+ const char* const restEnd = restStart + restSize;
+ for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
+ storedSkips += (unsigned) (restPtr - restStart);
+ if (restPtr != restEnd)
+ {
+ size_t sizeCheck;
+ int seekResult = fseek(file, storedSkips, SEEK_CUR);
+ if (seekResult) EXM_THROW(68, "Skip error (end of block)");
+ storedSkips = 0;
+ sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
+ if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block");
+ }
+ }
+
+ return storedSkips;
+}
+
+static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
+{
+ char lastZeroByte[1] = { 0 };
+
+ if (storedSkips>0) /* implies g_sparseFileSupport */
+ {
+ int seekResult;
+ size_t sizeCheck;
+ storedSkips --;
+ seekResult = fseek(file, storedSkips, SEEK_CUR);
+ if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
+ sizeCheck = fwrite(lastZeroByte, 1, 1, file);
+ if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
+ }
+}
+
+
+static unsigned g_magicRead = 0;
+static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
{
unsigned long long filesize = 0;
char* in_buff;
char* out_buff;
+ unsigned storedSkips = 0;
/* Allocate Memory */
in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
@@ -556,7 +742,7 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE))
{ /* Cannot read next block : maybe new stream ? */
- fseek(finput, -4, SEEK_CUR);
+ g_magicRead = blockSize;
break;
}
@@ -570,10 +756,11 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
filesize += decodeSize;
/* Write Block */
- sizeCheck = fwrite(out_buff, 1, decodeSize, foutput);
- if (sizeCheck != (size_t)decodeSize) EXM_THROW(54, "Write error : cannot write decoded block into output\n");
+ storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips);
}
+ LZ4IO_fwriteSparseEnd(foutput, storedSkips);
+
/* Free */
free(in_buff);
free(out_buff);
@@ -582,172 +769,129 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
}
-static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
+
+typedef struct {
+ void* srcBuffer;
+ size_t srcBufferSize;
+ void* dstBuffer;
+ size_t dstBufferSize;
+ LZ4F_decompressionContext_t dCtx;
+} dRess_t;
+
+static const size_t LZ4IO_dBufferSize = 64 KB;
+
+static dRess_t LZ4IO_createDResources(void)
{
- unsigned long long filesize = 0;
- void* inBuff;
- void* outBuff;
-# define HEADERMAX 20
- char headerBuff[HEADERMAX];
- size_t sizeCheck;
- const size_t inBuffSize = 256 KB;
- const size_t outBuffSize = 256 KB;
- LZ4F_decompressionContext_t ctx;
+ dRess_t ress;
LZ4F_errorCode_t errorCode;
- unsigned storedSkips = 0;
/* init */
- errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create context : %s", LZ4F_getErrorName(errorCode));
- LZ4IO_writeLE32(headerBuff, LZ4IO_MAGICNUMBER); /* regenerated here, as it was already read from finput */
+ errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION);
+ if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
/* Allocate Memory */
- inBuff = malloc(256 KB);
- outBuff = malloc(256 KB);
- if (!inBuff || !outBuff) EXM_THROW(61, "Allocation error : not enough memory");
+ ress.srcBufferSize = LZ4IO_dBufferSize;
+ ress.srcBuffer = malloc(ress.srcBufferSize);
+ ress.dstBufferSize = LZ4IO_dBufferSize;
+ ress.dstBuffer = malloc(ress.dstBufferSize);
+ if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
+
+ return ress;
+}
+
+static void LZ4IO_freeDResources(dRess_t ress)
+{
+ LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx);
+ if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
+ free(ress.srcBuffer);
+ free(ress.dstBuffer);
+}
- /* Init feed with magic number (already consumed from FILE) */
+
+static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile)
+{
+ unsigned long long filesize = 0;
+ LZ4F_errorCode_t nextToLoad;
+ unsigned storedSkips = 0;
+
+ /* Init feed with magic number (already consumed from FILE* sFile) */
{
- size_t inSize = 4;
- size_t outSize=0;
- LZ4IO_writeLE32(inBuff, LZ4IO_MAGICNUMBER);
- errorCode = LZ4F_decompress(ctx, outBuff, &outSize, inBuff, &inSize, NULL);
- if (LZ4F_isError(errorCode)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(errorCode));
+ size_t inSize = MAGICNUMBER_SIZE;
+ size_t outSize= 0;
+ LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER);
+ nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL);
+ if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
}
-
/* Main Loop */
- for (;;)
+ for (;nextToLoad;)
{
size_t readSize;
size_t pos = 0;
+ size_t decodedBytes = ress.dstBufferSize;
/* Read input */
- readSize = fread(inBuff, 1, inBuffSize, finput);
- if (!readSize) break; /* empty file or stream */
+ if (nextToLoad > ress.srcBufferSize) nextToLoad = ress.srcBufferSize;
+ readSize = fread(ress.srcBuffer, 1, nextToLoad, srcFile);
+ if (!readSize)
+ break; /* empty file or stream */
- while (pos < readSize)
+ while ((pos < readSize) || (decodedBytes == ress.dstBufferSize)) /* still to read, or still to flush */
{
/* Decode Input (at least partially) */
size_t remaining = readSize - pos;
- size_t decodedBytes = outBuffSize;
- errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, (char*)inBuff+pos, &remaining, NULL);
- if (LZ4F_isError(errorCode)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(errorCode));
+ decodedBytes = ress.dstBufferSize;
+ nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL);
+ if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
pos += remaining;
+ if (!nextToLoad) break;
if (decodedBytes)
{
/* Write Block */
filesize += decodedBytes;
- if (g_sparseFileSupport)
- {
- size_t* const oBuffStartT = (size_t*)outBuff; /* since outBuff is malloc'ed, it's aligned on size_t */
- size_t* oBuffPosT = oBuffStartT;
- size_t oBuffSizeT = decodedBytes / sizeT;
- size_t* const oBuffEndT = oBuffStartT + oBuffSizeT;
- static const size_t bs0T = (32 KB) / sizeT;
- while (oBuffPosT < oBuffEndT)
- {
- size_t seg0SizeT = bs0T;
- size_t nb0T;
- int seekResult;
- if (seg0SizeT > oBuffSizeT) seg0SizeT = oBuffSizeT;
- oBuffSizeT -= seg0SizeT;
- for (nb0T=0; (nb0T < seg0SizeT) && (oBuffPosT[nb0T] == 0); nb0T++) ;
- storedSkips += (unsigned)(nb0T * sizeT);
- if (storedSkips > 1 GB) /* avoid int overflow */
- {
- seekResult = fseek(foutput, 1 GB, SEEK_CUR);
- if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)");
- storedSkips -= 1 GB;
- }
- if (nb0T != seg0SizeT) /* not all 0s */
- {
- seekResult = fseek(foutput, storedSkips, SEEK_CUR);
- if (seekResult) EXM_THROW(68, "Skip error (sparse file)");
- storedSkips = 0;
- seg0SizeT -= nb0T;
- oBuffPosT += nb0T;
- sizeCheck = fwrite(oBuffPosT, sizeT, seg0SizeT, foutput);
- if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block");
- }
- oBuffPosT += seg0SizeT;
- }
- if (decodedBytes & maskT) /* size not multiple of sizeT (necessarily end of block) */
- {
- const char* const restStart = (char*)oBuffEndT;
- const char* restPtr = restStart;
- size_t restSize = decodedBytes & maskT;
- const char* const restEnd = restStart + restSize;
- for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
- storedSkips += (unsigned) (restPtr - restStart);
- if (restPtr != restEnd)
- {
- int seekResult = fseek(foutput, storedSkips, SEEK_CUR);
- if (seekResult) EXM_THROW(68, "Skip error (end of block)");
- storedSkips = 0;
- sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, foutput);
- if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block");
- }
- }
- }
- else
- {
- sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput);
- if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block");
- }
+ DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
+ storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips);
}
}
-
}
- if ((g_sparseFileSupport) && (storedSkips>0))
- {
- int seekResult;
- storedSkips --;
- seekResult = fseek(foutput, storedSkips, SEEK_CUR);
- if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
- memset(outBuff, 0, 1);
- sizeCheck = fwrite(outBuff, 1, 1, foutput);
- if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
- }
+ LZ4IO_fwriteSparseEnd(dstFile, storedSkips);
- /* Free */
- free(inBuff);
- free(outBuff);
- errorCode = LZ4F_freeDecompressionContext(ctx);
- if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
+ if (nextToLoad!=0)
+ EXM_THROW(67, "Unfinished stream");
return filesize;
}
-static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char U32store[MAGICNUMBER_SIZE])
+static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE])
{
void* buffer = malloc(64 KB);
size_t read = 1, sizeCheck;
unsigned long long total = MAGICNUMBER_SIZE;
+ unsigned storedSkips = 0;
- sizeCheck = fwrite(U32store, 1, MAGICNUMBER_SIZE, foutput);
- if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through error at start");
+ sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput);
+ if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error");
while (read)
{
read = fread(buffer, 1, 64 KB, finput);
total += read;
- sizeCheck = fwrite(buffer, 1, read, foutput);
- if (sizeCheck != read) EXM_THROW(50, "Pass-through error");
+ storedSkips = LZ4IO_fwriteSparse(foutput, buffer, read, storedSkips);
}
+ LZ4IO_fwriteSparseEnd(foutput, storedSkips);
free(buffer);
return total;
}
#define ENDOFSTREAM ((unsigned long long)-1)
-static unsigned long long selectDecoder( FILE* finput, FILE* foutput)
+static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput)
{
- unsigned char U32store[MAGICNUMBER_SIZE];
+ unsigned char MNstore[MAGICNUMBER_SIZE];
unsigned magicNumber, size;
int errorNb;
size_t nbReadBytes;
@@ -757,33 +901,41 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput)
nbCalls++;
/* Check Archive Header */
- nbReadBytes = fread(U32store, 1, MAGICNUMBER_SIZE, finput);
- if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */
- if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
- magicNumber = LZ4IO_readLE32(U32store); /* Little Endian format */
+ if (g_magicRead)
+ {
+ magicNumber = g_magicRead;
+ g_magicRead = 0;
+ }
+ else
+ {
+ nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
+ if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */
+ if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
+ magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */
+ }
if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */
switch(magicNumber)
{
case LZ4IO_MAGICNUMBER:
- return DEFAULT_DECOMPRESSOR(finput, foutput);
+ return LZ4IO_decompressLZ4F(ress, finput, foutput);
case LEGACY_MAGICNUMBER:
DISPLAYLEVEL(4, "Detected : Legacy format \n");
- return decodeLegacyStream(finput, foutput);
+ return LZ4IO_decodeLegacyStream(finput, foutput);
case LZ4IO_SKIPPABLE0:
DISPLAYLEVEL(4, "Skipping detected skippable area \n");
- nbReadBytes = fread(U32store, 1, 4, finput);
+ nbReadBytes = fread(MNstore, 1, 4, finput);
if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");
- size = LZ4IO_readLE32(U32store); /* Little Endian format */
+ size = LZ4IO_readLE32(MNstore); /* Little Endian format */
errorNb = fseek(finput, size, SEEK_CUR);
if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");
- return selectDecoder(finput, foutput);
+ return selectDecoder(ress, finput, foutput);
EXTENDED_FORMAT;
default:
if (nbCalls == 1) /* just started */
{
if (g_overwrite)
- return LZ4IO_passThrough(finput, foutput, U32store);
+ return LZ4IO_passThrough(finput, foutput, MNstore);
EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */
}
DISPLAYLEVEL(2, "Stream followed by unrecognized data\n");
@@ -792,43 +944,95 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput)
}
-int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename)
+static int LZ4IO_decompressFile_extRess(dRess_t ress, const char* input_filename, const char* output_filename)
{
unsigned long long filesize = 0, decodedSize=0;
FILE* finput;
FILE* foutput;
- clock_t start, end;
/* Init */
- start = clock();
- get_fileHandle(input_filename, output_filename, &finput, &foutput);
+ if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput))
+ return 1;
/* sparse file */
- if (g_sparseFileSupport && foutput) { SET_SPARSE_FILE_MODE(foutput); }
+ if (g_sparseFileSupport) { SET_SPARSE_FILE_MODE(foutput); }
/* Loop over multiple streams */
do
{
- decodedSize = selectDecoder(finput, foutput);
+ decodedSize = selectDecoder(ress, finput, foutput);
if (decodedSize != ENDOFSTREAM)
filesize += decodedSize;
} while (decodedSize != ENDOFSTREAM);
/* Final Status */
- end = clock();
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
- {
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;
- DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
- }
/* Close */
fclose(finput);
fclose(foutput);
- /* Error status = OK */
return 0;
}
+
+int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename)
+{
+ dRess_t ress;
+ clock_t start, end;
+ int missingFiles = 0;
+
+ start = clock();
+
+ ress = LZ4IO_createDResources();
+ missingFiles += LZ4IO_decompressFile_extRess(ress, input_filename, output_filename);
+ LZ4IO_freeDResources(ress);
+
+ end = clock();
+ if (end==start) end=start+1;
+ {
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds);
+ }
+
+ return missingFiles;
+}
+
+
+int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix)
+{
+ int i;
+ int skippedFiles = 0;
+ int missingFiles = 0;
+ char* outFileName = (char*)malloc(FNSPACE);
+ size_t ofnSize = FNSPACE;
+ const size_t suffixSize = strlen(suffix);
+ char* ifnSuffix = (char*)malloc(suffixSize + 1);
+ dRess_t ress;
+
+ ress = LZ4IO_createDResources();
+
+ for (i=0; i<ifntSize; i++)
+ {
+ size_t ifnSize = strlen(inFileNamesTable[i]);
+ strcpy(ifnSuffix, inFileNamesTable[i] + ifnSize - suffixSize);
+ if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); }
+ if (ifnSize <= suffixSize || strcmp(ifnSuffix, suffix) != 0)
+ {
+ DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]);
+ skippedFiles++;
+ continue;
+ }
+ memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize);
+ outFileName[ifnSize-suffixSize] = '\0';
+
+ missingFiles += LZ4IO_decompressFile_extRess(ress, inFileNamesTable[i], outFileName);
+ }
+
+ LZ4IO_freeDResources(ress);
+ free(outFileName);
+ free(ifnSuffix);
+ return missingFiles + skippedFiles;
+}
diff --git a/programs/lz4io.h b/programs/lz4io.h
index 75a36e1..11c8fdb 100644
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -51,8 +51,9 @@ static char const nulmark[] = "/dev/null";
int LZ4IO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel);
int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename);
-int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel);
+int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel);
+int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix);
/* ************************************************** */
/* ****************** Parameters ******************** */