diff options
| -rw-r--r-- | lib/lz4hc.c | 14 | ||||
| -rw-r--r-- | lib/lz4opt.h | 61 | ||||
| -rw-r--r-- | programs/lz4.1 | 2 | ||||
| -rw-r--r-- | programs/lz4.1.md | 5 | ||||
| -rw-r--r-- | tests/Makefile | 20 | ||||
| -rw-r--r-- | tests/fuzzer.c | 59 |
6 files changed, 96 insertions, 65 deletions
diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 2fff29b..f2c2566 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -428,10 +428,7 @@ static int LZ4HC_compress_hashChain ( /* init */ *srcSizePtr = 0; - if (limit == limitedDestSize && maxOutputSize < 1) return 0; /* Impossible to store anything */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - - if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support limitations LZ4 decompressor */ + if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ /* Main Loop */ @@ -648,11 +645,12 @@ static int LZ4HC_compress_generic ( { lz4opt,8192, LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ }; + if (limit == limitedDestSize && dstCapacity < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ + ctx->end += *srcSizePtr; if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe something to review */ cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel); - if (limit == limitedDestSize) - cLevel = MIN(LZ4HC_CLEVEL_OPT_MIN-1, cLevel); /* no limitedDestSize variant for lz4opt */ assert(cLevel >= 0); assert(cLevel <= LZ4HC_CLEVEL_MAX); { cParams_t const cParam = clTable[cLevel]; @@ -662,8 +660,8 @@ static int LZ4HC_compress_generic ( cParam.nbSearches, limit); assert(cParam.strat == lz4opt); return LZ4HC_compress_optimal(ctx, - src, dst, *srcSizePtr, dstCapacity, limit, - cParam.nbSearches, cParam.targetLength, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, cParam.targetLength, limit, cLevel == LZ4HC_CLEVEL_MAX); /* ultra mode */ } } diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 6c15598..5a8438c 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -99,11 +99,11 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, const char* const source, char* dst, - int inputSize, + int* srcSizePtr, int dstCapacity, - limitedOutput_directive limit, int const nbSearches, size_t sufficient_len, + limitedOutput_directive limit, int const fullUpdate ) { @@ -112,14 +112,17 @@ static int LZ4HC_compress_optimal ( const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; - const BYTE* const iend = ip + inputSize; + const BYTE* const iend = ip + *srcSizePtr; const BYTE* const mflimit = iend - MFLIMIT; const BYTE* const matchlimit = iend - LASTLITERALS; BYTE* op = (BYTE*) dst; - BYTE* const oend = op + dstCapacity; + BYTE* opSaved = (BYTE*) dst; + BYTE* oend = op + dstCapacity; /* init */ DEBUGLOG(5, "LZ4HC_compress_optimal"); + *srcSizePtr = 0; + if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; /* Main Loop */ @@ -136,8 +139,9 @@ static int LZ4HC_compress_optimal ( /* good enough solution : immediate encoding */ int const firstML = firstMatch.len; const BYTE* const matchPos = ip - firstMatch.off; + opSaved = op; if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ - return 0; /* error */ + goto _dest_overflow; continue; } @@ -306,26 +310,47 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */ rPos += ml; assert(ml >= MINMATCH); assert((offset >= 1) && (offset <= MAX_DISTANCE)); + opSaved = op; if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ - return 0; /* error */ + goto _dest_overflow; } } } /* while (ip < mflimit) */ +_last_literals: /* Encode Last Literals */ - { int lastRun = (int)(iend - anchor); - if ( (limit) - && (((char*)op - dst) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)dstCapacity)) - return 0; /* Check output limit */ - if (lastRun >= (int)RUN_MASK) { - *op++=(RUN_MASK<<ML_BITS); - lastRun-=RUN_MASK; - for (; lastRun > 254 ; lastRun-=255) *op++ = 255; - *op++ = (BYTE) lastRun; - } else *op++ = (BYTE)(lastRun<<ML_BITS); - memcpy(op, anchor, iend - anchor); - op += iend-anchor; + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + litLength + lastRunSize; + if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) return 0; /* Check output limit */ + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (size_t)(oend - op) - 1; + litLength = (lastRunSize + 255 - RUN_MASK) / 255; + lastRunSize -= litLength; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + memcpy(op, anchor, lastRunSize); + op += lastRunSize; } /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); return (int) ((char*)op-dst); + +_dest_overflow: + if (limit == limitedDestSize) { + op = opSaved; /* restore correct out pointer */ + goto _last_literals; + } + return 0; } diff --git a/programs/lz4.1 b/programs/lz4.1 index 39d78cd..7ce5da0 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -35,7 +35,7 @@ When writing scripts that need to decompress files, it is recommended to always \fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression) . .IP "\(bu" 4 -\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silent them) +\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them) . .IP "\(bu" 4 If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\. diff --git a/programs/lz4.1.md b/programs/lz4.1.md index c6b99bc..b7b8570 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -37,7 +37,7 @@ Differences are : * `lz4 file.lz4` will default to decompression (use `-z` to force compression) * `lz4` shows real-time notification statistics during compression or decompression of a single file - (use `-q` to silent them) + (use `-q` to silence them) * If no destination name is provided, result is sent to `stdout` _except if stdout is the console_. * If no destination name is provided, __and__ if `stdout` is the console, @@ -45,6 +45,9 @@ Differences are : * As a consequence of previous rules, note the following example : `lz4 file | consumer` sends compressed data to `consumer` through `stdout`, hence it does _not_ create `file.lz4`. + * Another consequence of those rules is that to run `lz4` under `nohup`, + you should provide a destination file: `nohup lz4 file file.lz4`, + because `nohup` writes the specified command's output to a file. Default behaviors can be modified by opt-in commands, detailed below. diff --git a/tests/Makefile b/tests/Makefile index 1a907b7..819ba43 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -35,11 +35,12 @@ PRGDIR := ../programs TESTDIR := versionsTest PYTHON ?= python3 +DEBUGFLAGS = -g -DLZ4_DEBUG=1 CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make -CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ +CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 -CFLAGS += $(MOREFLAGS) +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_ FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) @@ -77,21 +78,26 @@ lz4c unlz4 lz4cat: lz4 lz4c32: # create a 32-bits version for 32/64 interop tests $(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)" -fullbench : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c +%.o : $(LZ4DIR)/%.c $(LZ4DIR)/%.h + $(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@ + +fullbench : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) -fullbench-lib: fullbench.c $(LZ4DIR)/xxhash.c +$(LZ4DIR)/liblz4.a: $(MAKE) -C $(LZ4DIR) liblz4.a - $(CC) $(FLAGS) $^ -o $@$(EXT) $(LZ4DIR)/liblz4.a + +fullbench-lib: fullbench.c $(LZ4DIR)/liblz4.a + $(CC) $(FLAGS) $^ -o $@$(EXT) fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c $(MAKE) -C $(LZ4DIR) liblz4 $(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/liblz4.dll -fuzzer : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o fuzzer.c +fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) -frametest: $(LZ4DIR)/lz4frame.o $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o frametest.c +frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c $(CC) $(FLAGS) $^ -o $@$(EXT) datagen : $(PRGDIR)/datagen.c datagencli.c diff --git a/tests/fuzzer.c b/tests/fuzzer.c index ddd293c..c134fe3 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -345,10 +345,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } DISPLAYLEVEL(5, " OK \n"); - } - else + } else { DISPLAYLEVEL(5, " \n"); - } + } } /* Test compression HC destSize */ FUZ_DISPLAYTEST; @@ -359,11 +358,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed"); compressedBuffer[targetSize] = endCheck; ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel); + DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ", + compressionLevel, ret, targetSize, srcSize, blockSize); LZ4_freeHC(ctx); FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !"); FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !"); FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !"); - DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); + DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ", + compressionLevel, ret, targetSize, srcSize, blockSize); if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, srcSize, 0); @@ -380,10 +382,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } DISPLAYLEVEL(5, " OK \n"); - } - else + } else { DISPLAYLEVEL(5, " \n"); - } + } } /* Test compression HC */ FUZ_DISPLAYTEST; @@ -898,7 +899,6 @@ static void FUZ_unitTests(int compressionLevel) } dict = dst; - //dict = testInput + segStart; dictSize = segSize; dst += segSize + 1; @@ -972,28 +972,27 @@ static void FUZ_unitTests(int compressionLevel) #define BSIZE2 16435 /* first block */ - - messageSize = BSIZE1; - XXH64_update(&xxhOrig, testInput + iNext, messageSize); - crcOrig = XXH64_digest(&xxhOrig); - - result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - - result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); - - XXH64_update(&xxhNew, testVerify + dNext, messageSize); - { U64 const crcNew = XXH64_digest(&xxhNew); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } - - /* prepare next message */ - dNext += messageSize; - totalMessageSize += messageSize; - messageSize = BSIZE2; - iNext = 132000; - memcpy(testInput + iNext, testInput + 8, messageSize); - if (dNext > dBufferSize) dNext = 0; + messageSize = BSIZE1; + XXH64_update(&xxhOrig, testInput + iNext, messageSize); + crcOrig = XXH64_digest(&xxhOrig); + + result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); + + result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); + FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); + + XXH64_update(&xxhNew, testVerify + dNext, messageSize); + { U64 const crcNew = XXH64_digest(&xxhNew); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } + + /* prepare next message */ + dNext += messageSize; + totalMessageSize += messageSize; + messageSize = BSIZE2; + iNext = 132000; + memcpy(testInput + iNext, testInput + 8, messageSize); + if (dNext > dBufferSize) dNext = 0; while (totalMessageSize < 9 MB) { XXH64_update(&xxhOrig, testInput + iNext, messageSize); |